From f722bd2387df9163760014e9555928ec487ae95f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 00:53:20 +0200 Subject: [PATCH] Separate background jobs into different queues. ATTENTION: new queue "pull" must be added to the Sidekiq invokation in your systemd file The pull queue will handle link crawling, thread resolving, and OStatus processing. Such tasks are more likely to hang for a longer time (due to network requests) so it is more sensible to not make the "in-house" tasks wait for them. --- app/workers/after_remote_follow_request_worker.rb | 2 +- app/workers/after_remote_follow_worker.rb | 2 +- app/workers/import_worker.rb | 2 +- app/workers/link_crawl_worker.rb | 2 +- app/workers/merge_worker.rb | 2 ++ app/workers/notification_worker.rb | 2 +- app/workers/processing_worker.rb | 2 +- app/workers/regeneration_worker.rb | 2 ++ app/workers/salmon_worker.rb | 2 +- app/workers/thread_resolve_worker.rb | 2 +- app/workers/unmerge_worker.rb | 2 ++ docker-compose.yml | 2 +- docs/Running-Mastodon/Production-guide.md | 2 +- 13 files changed, 16 insertions(+), 10 deletions(-) diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb index f1d6869cc7..1f2db30615 100644 --- a/app/workers/after_remote_follow_request_worker.rb +++ b/app/workers/after_remote_follow_request_worker.rb @@ -3,7 +3,7 @@ class AfterRemoteFollowRequestWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'pull', retry: 5 def perform(follow_request_id) follow_request = FollowRequest.find(follow_request_id) diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb index 0d04456a94..bdd2c2a91d 100644 --- a/app/workers/after_remote_follow_worker.rb +++ b/app/workers/after_remote_follow_worker.rb @@ -3,7 +3,7 @@ class AfterRemoteFollowWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'pull', retry: 5 def perform(follow_id) follow = Follow.find(follow_id) diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb index a3ae2a85a4..7cf29fb53c 100644 --- a/app/workers/import_worker.rb +++ b/app/workers/import_worker.rb @@ -5,7 +5,7 @@ require 'csv' class ImportWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(import_id) import = Import.find(import_id) diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb index af3394b8b3..834b0088bd 100644 --- a/app/workers/link_crawl_worker.rb +++ b/app/workers/link_crawl_worker.rb @@ -3,7 +3,7 @@ class LinkCrawlWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(status_id) FetchLinkCardService.new.call(Status.find(status_id)) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 0f288f43fe..d745cb99c7 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -3,6 +3,8 @@ class MergeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull' + def perform(from_account_id, into_account_id) FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id)) end diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb index 1a2faefd8e..da1d6ab455 100644 --- a/app/workers/notification_worker.rb +++ b/app/workers/notification_worker.rb @@ -3,7 +3,7 @@ class NotificationWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'push', retry: 5 def perform(xml, source_account_id, target_account_id) SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id)) diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb index 5df404bcc9..4a467d9244 100644 --- a/app/workers/processing_worker.rb +++ b/app/workers/processing_worker.rb @@ -3,7 +3,7 @@ class ProcessingWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options queue: 'pull', backtrace: true def perform(account_id, body) ProcessFeedService.new.call(body, Account.find(account_id)) diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb index 3aece0ba28..289b63d84a 100644 --- a/app/workers/regeneration_worker.rb +++ b/app/workers/regeneration_worker.rb @@ -3,6 +3,8 @@ class RegenerationWorker include Sidekiq::Worker + sidekiq_options queue: 'pull', backtrace: true + def perform(account_id, timeline_type) PrecomputeFeedService.new.call(timeline_type, Account.find(account_id)) end diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb index fc95ce47ff..2888b574b6 100644 --- a/app/workers/salmon_worker.rb +++ b/app/workers/salmon_worker.rb @@ -3,7 +3,7 @@ class SalmonWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options queue: 'pull', backtrace: true def perform(account_id, body) ProcessInteractionService.new.call(body, Account.find(account_id)) diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb index 593edd032f..38287e8e64 100644 --- a/app/workers/thread_resolve_worker.rb +++ b/app/workers/thread_resolve_worker.rb @@ -3,7 +3,7 @@ class ThreadResolveWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(child_status_id, parent_url) child_status = Status.find(child_status_id) diff --git a/app/workers/unmerge_worker.rb b/app/workers/unmerge_worker.rb index dbf7243de7..ea6aacebf6 100644 --- a/app/workers/unmerge_worker.rb +++ b/app/workers/unmerge_worker.rb @@ -3,6 +3,8 @@ class UnmergeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull' + def perform(from_account_id, into_account_id) FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id)) end diff --git a/docker-compose.yml b/docker-compose.yml index 68c8ef960e..d6ba66ddeb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: restart: always build: . env_file: .env.production - command: bundle exec sidekiq -q default -q mailers -q push + command: bundle exec sidekiq -q default -q mailers -q pull -q push depends_on: - db - redis diff --git a/docs/Running-Mastodon/Production-guide.md b/docs/Running-Mastodon/Production-guide.md index f0dd7bd2b5..469fefa943 100644 --- a/docs/Running-Mastodon/Production-guide.md +++ b/docs/Running-Mastodon/Production-guide.md @@ -180,7 +180,7 @@ User=mastodon WorkingDirectory=/home/mastodon/live Environment="RAILS_ENV=production" Environment="DB_POOL=5" -ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q push +ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push TimeoutSec=15 Restart=always