From ad40bf5e0c77baacbeda248ccca8d310e786ff61 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 30 May 2018 21:09:30 +0200
Subject: [PATCH] Speed up some rake tasks by moving execution to Sidekiq
 (#7678)

* Speed up some rake tasks by moving execution to Sidekiq

mastodon:media:remove_silenced
mastodon:media:remove_remote
mastodon:media:redownload_avatars
mastodon:feeds:build

* Fix code style issue
---
 .../maintenance/destroy_media_worker.rb       | 14 +++++++++++
 .../redownload_account_media_worker.rb        | 16 ++++++++++++
 .../maintenance/uncache_media_worker.rb       | 18 +++++++++++++
 lib/tasks/mastodon.rake                       | 25 +++++++------------
 4 files changed, 57 insertions(+), 16 deletions(-)
 create mode 100644 app/workers/maintenance/destroy_media_worker.rb
 create mode 100644 app/workers/maintenance/redownload_account_media_worker.rb
 create mode 100644 app/workers/maintenance/uncache_media_worker.rb

diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb
new file mode 100644
index 0000000000..5f052983be
--- /dev/null
+++ b/app/workers/maintenance/destroy_media_worker.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Maintenance::DestroyMediaWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(media_attachment_id)
+    media = MediaAttachment.find(media_attachment_id)
+    media.destroy
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb
new file mode 100644
index 0000000000..fc26815f25
--- /dev/null
+++ b/app/workers/maintenance/redownload_account_media_worker.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class Maintenance::RedownloadAccountMediaWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull', retry: false
+
+  def perform(account_id)
+    account = Account.find(account_id)
+    account.reset_avatar!
+    account.reset_header!
+    account.save
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb
new file mode 100644
index 0000000000..f6a51a1b8b
--- /dev/null
+++ b/app/workers/maintenance/uncache_media_worker.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Maintenance::UncacheMediaWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(media_attachment_id)
+    media = MediaAttachment.find(media_attachment_id)
+
+    return unless media.file.exists?
+
+    media.file.destroy
+    media.save
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index 00a85fa5e8..8ff29ea9e3 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -502,18 +502,17 @@ namespace :mastodon do
 
     desc 'Remove media attachments attributed to silenced accounts'
     task remove_silenced: :environment do
-      MediaAttachment.where(account: Account.silenced).find_each(&:destroy)
+      MediaAttachment.where(account: Account.silenced).select(:id).find_in_batches do |media_attachments|
+        Maintenance::DestroyMediaWorker.push_bulk(media_attachments.map(&:id))
+      end
     end
 
     desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
     task remove_remote: :environment do
       time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
 
-      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).find_each do |media|
-        next unless media.file.exists?
-
-        media.file.destroy
-        media.save
+      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).find_in_batches do |media_attachments|
+        Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
       end
     end
 
@@ -529,14 +528,8 @@ namespace :mastodon do
       accounts = Account.remote
       accounts = accounts.where(domain: ENV['DOMAIN']) if ENV['DOMAIN'].present?
 
-      accounts.find_each do |account|
-        begin
-          account.reset_avatar!
-          account.reset_header!
-          account.save
-        rescue Paperclip::Error
-          puts "Error resetting avatar and header for account #{username}@#{domain}"
-        end
+      accounts.select(:id).find_in_batches do |accounts_batch|
+        Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts_batch.map(&:id))
       end
     end
   end
@@ -568,8 +561,8 @@ namespace :mastodon do
 
     desc 'Generates home timelines for users who logged in in the past two weeks'
     task build: :environment do
-      User.active.includes(:account).find_each do |u|
-        PrecomputeFeedService.new.call(u.account)
+      User.active.select(:account_id).find_in_batches do |users|
+        RegenerationWorker.push_bulk(users.map(&:account_id))
       end
     end
   end