From 5c9abdeff1d0cf3e14d84c5ae298e6a5beccaf18 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 27 Sep 2022 03:08:19 +0200
Subject: [PATCH] Add retention policy for cached content and media (#19232)

---
 app/lib/redis_configuration.rb                |  6 +-
 app/lib/vacuum.rb                             |  3 +
 .../vacuum/access_tokens_vacuum.rb}           | 17 ++++--
 app/lib/vacuum/backups_vacuum.rb              | 25 +++++++++
 app/lib/vacuum/feeds_vacuum.rb                | 34 +++++++++++
 app/lib/vacuum/media_attachments_vacuum.rb    | 40 +++++++++++++
 app/lib/vacuum/preview_cards_vacuum.rb        | 39 +++++++++++++
 app/lib/vacuum/statuses_vacuum.rb             | 54 ++++++++++++++++++
 app/lib/vacuum/system_keys_vacuum.rb          | 13 +++++
 app/models/content_retention_policy.rb        | 25 +++++++++
 app/models/form/admin_settings.rb             |  4 ++
 app/views/admin/settings/edit.html.haml       |  8 ++-
 .../scheduler/backup_cleanup_scheduler.rb     | 17 ------
 .../scheduler/feed_cleanup_scheduler.rb       | 35 ------------
 .../scheduler/media_cleanup_scheduler.rb      | 17 ------
 app/workers/scheduler/vacuum_scheduler.rb     | 56 +++++++++++++++++++
 config/locales/simple_form.en.yml             |  8 +++
 config/settings.yml                           |  1 +
 config/sidekiq.yml                            | 16 +-----
 spec/fabricators/access_grant_fabricator.rb   |  6 ++
 spec/fabricators/preview_card_fabricator.rb   |  1 +
 spec/lib/vacuum/access_tokens_vacuum_spec.rb  | 33 +++++++++++
 spec/lib/vacuum/backups_vacuum_spec.rb        | 24 ++++++++
 spec/lib/vacuum/feeds_vacuum_spec.rb          | 30 ++++++++++
 .../vacuum/media_attachments_vacuum_spec.rb   | 47 ++++++++++++++++
 spec/lib/vacuum/preview_cards_vacuum_spec.rb  | 36 ++++++++++++
 spec/lib/vacuum/statuses_vacuum_spec.rb       | 36 ++++++++++++
 spec/lib/vacuum/system_keys_vacuum_spec.rb    | 22 ++++++++
 .../scheduler/feed_cleanup_scheduler_spec.rb  | 26 ---------
 .../scheduler/media_cleanup_scheduler_spec.rb | 15 -----
 30 files changed, 559 insertions(+), 135 deletions(-)
 create mode 100644 app/lib/vacuum.rb
 rename app/{workers/scheduler/doorkeeper_cleanup_scheduler.rb => lib/vacuum/access_tokens_vacuum.rb} (56%)
 create mode 100644 app/lib/vacuum/backups_vacuum.rb
 create mode 100644 app/lib/vacuum/feeds_vacuum.rb
 create mode 100644 app/lib/vacuum/media_attachments_vacuum.rb
 create mode 100644 app/lib/vacuum/preview_cards_vacuum.rb
 create mode 100644 app/lib/vacuum/statuses_vacuum.rb
 create mode 100644 app/lib/vacuum/system_keys_vacuum.rb
 create mode 100644 app/models/content_retention_policy.rb
 delete mode 100644 app/workers/scheduler/backup_cleanup_scheduler.rb
 delete mode 100644 app/workers/scheduler/feed_cleanup_scheduler.rb
 delete mode 100644 app/workers/scheduler/media_cleanup_scheduler.rb
 create mode 100644 app/workers/scheduler/vacuum_scheduler.rb
 create mode 100644 spec/fabricators/access_grant_fabricator.rb
 create mode 100644 spec/lib/vacuum/access_tokens_vacuum_spec.rb
 create mode 100644 spec/lib/vacuum/backups_vacuum_spec.rb
 create mode 100644 spec/lib/vacuum/feeds_vacuum_spec.rb
 create mode 100644 spec/lib/vacuum/media_attachments_vacuum_spec.rb
 create mode 100644 spec/lib/vacuum/preview_cards_vacuum_spec.rb
 create mode 100644 spec/lib/vacuum/statuses_vacuum_spec.rb
 create mode 100644 spec/lib/vacuum/system_keys_vacuum_spec.rb
 delete mode 100644 spec/workers/scheduler/feed_cleanup_scheduler_spec.rb
 delete mode 100644 spec/workers/scheduler/media_cleanup_scheduler_spec.rb

diff --git a/app/lib/redis_configuration.rb b/app/lib/redis_configuration.rb
index e14d6c8b670..f0e86d985b8 100644
--- a/app/lib/redis_configuration.rb
+++ b/app/lib/redis_configuration.rb
@@ -7,9 +7,7 @@ class RedisConfiguration
       @pool = ConnectionPool.new(size: new_pool_size) { new.connection }
     end
 
-    def with
-      pool.with { |redis| yield redis }
-    end
+    delegate :with, to: :pool
 
     def pool
       @pool ||= establish_pool(pool_size)
@@ -17,7 +15,7 @@ class RedisConfiguration
 
     def pool_size
       if Sidekiq.server?
-        Sidekiq.options[:concurrency]
+        Sidekiq[:concurrency]
       else
         ENV['MAX_THREADS'] || 5
       end
diff --git a/app/lib/vacuum.rb b/app/lib/vacuum.rb
new file mode 100644
index 00000000000..9db1ec90bed
--- /dev/null
+++ b/app/lib/vacuum.rb
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+module Vacuum; end
diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/lib/vacuum/access_tokens_vacuum.rb
similarity index 56%
rename from app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
rename to app/lib/vacuum/access_tokens_vacuum.rb
index 9303a352fb7..4f3878027ab 100644
--- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
+++ b/app/lib/vacuum/access_tokens_vacuum.rb
@@ -1,13 +1,18 @@
 # frozen_string_literal: true
 
-class Scheduler::DoorkeeperCleanupScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options retry: 0
-
+class Vacuum::AccessTokensVacuum
   def perform
+    vacuum_revoked_access_tokens!
+    vacuum_revoked_access_grants!
+  end
+
+  private
+
+  def vacuum_revoked_access_tokens!
     Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
+  end
+
+  def vacuum_revoked_access_grants!
     Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all
-    SystemKey.expired.delete_all
   end
 end
diff --git a/app/lib/vacuum/backups_vacuum.rb b/app/lib/vacuum/backups_vacuum.rb
new file mode 100644
index 00000000000..3b83072f369
--- /dev/null
+++ b/app/lib/vacuum/backups_vacuum.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class Vacuum::BackupsVacuum
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_expired_backups! if retention_period?
+  end
+
+  private
+
+  def vacuum_expired_backups!
+    backups_past_retention_period.in_batches.destroy_all
+  end
+
+  def backups_past_retention_period
+    Backup.unscoped.where(Backup.arel_table[:created_at].lt(@retention_period.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb
new file mode 100644
index 00000000000..f46bcf75f7f
--- /dev/null
+++ b/app/lib/vacuum/feeds_vacuum.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class Vacuum::FeedsVacuum
+  def perform
+    vacuum_inactive_home_feeds!
+    vacuum_inactive_list_feeds!
+  end
+
+  private
+
+  def vacuum_inactive_home_feeds!
+    inactive_users.select(:id, :account_id).find_in_batches do |users|
+      feed_manager.clean_feeds!(:home, users.map(&:account_id))
+    end
+  end
+
+  def vacuum_inactive_list_feeds!
+    inactive_users_lists.select(:id).find_in_batches do |lists|
+      feed_manager.clean_feeds!(:list, lists.map(&:id))
+    end
+  end
+
+  def inactive_users
+    User.confirmed.inactive
+  end
+
+  def inactive_users_lists
+    List.where(account_id: inactive_users.select(:account_id))
+  end
+
+  def feed_manager
+    FeedManager.instance
+  end
+end
diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb
new file mode 100644
index 00000000000..7fb347ce4d5
--- /dev/null
+++ b/app/lib/vacuum/media_attachments_vacuum.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Vacuum::MediaAttachmentsVacuum
+  TTL = 1.day.freeze
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_cached_files! if retention_period?
+    vacuum_orphaned_records!
+  end
+
+  private
+
+  def vacuum_cached_files!
+    media_attachments_past_retention_period.find_each do |media_attachment|
+      media_attachment.file.destroy
+      media_attachment.thumbnail.destroy
+      media_attachment.save
+    end
+  end
+
+  def vacuum_orphaned_records!
+    orphaned_media_attachments.in_batches.destroy_all
+  end
+
+  def media_attachments_past_retention_period
+    MediaAttachment.unscoped.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago))
+  end
+
+  def orphaned_media_attachments
+    MediaAttachment.unscoped.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb
new file mode 100644
index 00000000000..84ef100ed94
--- /dev/null
+++ b/app/lib/vacuum/preview_cards_vacuum.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Vacuum::PreviewCardsVacuum
+  TTL = 1.day.freeze
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_cached_images! if retention_period?
+    vacuum_orphaned_records!
+  end
+
+  private
+
+  def vacuum_cached_images!
+    preview_cards_past_retention_period.find_each do |preview_card|
+      preview_card.image.destroy
+      preview_card.save
+    end
+  end
+
+  def vacuum_orphaned_records!
+    orphaned_preview_cards.in_batches.destroy_all
+  end
+
+  def preview_cards_past_retention_period
+    PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago))
+  end
+
+  def orphaned_preview_cards
+    PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago))
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb
new file mode 100644
index 00000000000..41d6ba270cc
--- /dev/null
+++ b/app/lib/vacuum/statuses_vacuum.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class Vacuum::StatusesVacuum
+  include Redisable
+
+  def initialize(retention_period)
+    @retention_period = retention_period
+  end
+
+  def perform
+    vacuum_statuses! if retention_period?
+  end
+
+  private
+
+  def vacuum_statuses!
+    statuses_scope.find_in_batches do |statuses|
+      # Side-effects not covered by foreign keys, such
+      # as the search index, must be handled first.
+
+      remove_from_account_conversations(statuses)
+      remove_from_search_index(statuses)
+
+      # Foreign keys take care of most associated records
+      # for us. Media attachments will be orphaned.
+
+      Status.where(id: statuses.map(&:id)).delete_all
+    end
+  end
+
+  def statuses_scope
+    Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility)
+  end
+
+  def retention_period_as_id
+    Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false)
+  end
+
+  def analyze_statuses!
+    ActiveRecord::Base.connection.execute('ANALYZE statuses')
+  end
+
+  def remove_from_account_conversations(statuses)
+    Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations)
+  end
+
+  def remove_from_search_index(statuses)
+    with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled?
+  end
+
+  def retention_period?
+    @retention_period.present?
+  end
+end
diff --git a/app/lib/vacuum/system_keys_vacuum.rb b/app/lib/vacuum/system_keys_vacuum.rb
new file mode 100644
index 00000000000..ceee2fd164b
--- /dev/null
+++ b/app/lib/vacuum/system_keys_vacuum.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Vacuum::SystemKeysVacuum
+  def perform
+    vacuum_expired_system_keys!
+  end
+
+  private
+
+  def vacuum_expired_system_keys!
+    SystemKey.expired.delete_all
+  end
+end
diff --git a/app/models/content_retention_policy.rb b/app/models/content_retention_policy.rb
new file mode 100644
index 00000000000..b5e922c8c5f
--- /dev/null
+++ b/app/models/content_retention_policy.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class ContentRetentionPolicy
+  def self.current
+    new
+  end
+
+  def media_cache_retention_period
+    retention_period Setting.media_cache_retention_period
+  end
+
+  def content_cache_retention_period
+    retention_period Setting.content_cache_retention_period
+  end
+
+  def backups_retention_period
+    retention_period Setting.backups_retention_period
+  end
+
+  private
+
+  def retention_period(value)
+    value.days if value.is_a?(Integer) && value.positive?
+  end
+end
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 97fabc6ac97..3a71509162c 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -32,6 +32,9 @@ class Form::AdminSettings
     show_domain_blocks_rationale
     noindex
     require_invite_text
+    media_cache_retention_period
+    content_cache_retention_period
+    backups_retention_period
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -64,6 +67,7 @@ class Form::AdminSettings
   validates :bootstrap_timeline_accounts, existing_username: { multiple: true }
   validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }
   validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }
+  validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true }
 
   def initialize(_attributes = {})
     super
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 64687b7a60f..1dfd216439d 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -45,7 +45,6 @@
 
   .fields-group
     = f.input :require_invite_text, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.require_invite_text.title'), hint: t('admin.settings.registrations.require_invite_text.desc_html'), disabled: !approved_registrations?
-  .fields-group
 
   %hr.spacer/
 
@@ -100,5 +99,12 @@
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
+  %hr.spacer/
+
+  .fields-group
+    = f.input :media_cache_retention_period, wrapper: :with_block_label, input_html: { pattern: '[0-9]+' }
+    = f.input :content_cache_retention_period, wrapper: :with_block_label, input_html: { pattern: '[0-9]+' }
+    = f.input :backups_retention_period, wrapper: :with_block_label, input_html: { pattern: '[0-9]+' }
+
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb
deleted file mode 100644
index 85d5312c0ad..00000000000
--- a/app/workers/scheduler/backup_cleanup_scheduler.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::BackupCleanupScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options retry: 0
-
-  def perform
-    old_backups.reorder(nil).find_each(&:destroy!)
-  end
-
-  private
-
-  def old_backups
-    Backup.where('created_at < ?', 7.days.ago)
-  end
-end
diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb
deleted file mode 100644
index aa0cc8b8df3..00000000000
--- a/app/workers/scheduler/feed_cleanup_scheduler.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::FeedCleanupScheduler
-  include Sidekiq::Worker
-  include Redisable
-
-  sidekiq_options retry: 0
-
-  def perform
-    clean_home_feeds!
-    clean_list_feeds!
-  end
-
-  private
-
-  def clean_home_feeds!
-    feed_manager.clean_feeds!(:home, inactive_account_ids)
-  end
-
-  def clean_list_feeds!
-    feed_manager.clean_feeds!(:list, inactive_list_ids)
-  end
-
-  def inactive_account_ids
-    @inactive_account_ids ||= User.confirmed.inactive.pluck(:account_id)
-  end
-
-  def inactive_list_ids
-    List.where(account_id: inactive_account_ids).pluck(:id)
-  end
-
-  def feed_manager
-    FeedManager.instance
-  end
-end
diff --git a/app/workers/scheduler/media_cleanup_scheduler.rb b/app/workers/scheduler/media_cleanup_scheduler.rb
deleted file mode 100644
index 24d30a6be59..00000000000
--- a/app/workers/scheduler/media_cleanup_scheduler.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-class Scheduler::MediaCleanupScheduler
-  include Sidekiq::Worker
-
-  sidekiq_options retry: 0
-
-  def perform
-    unattached_media.find_each(&:destroy)
-  end
-
-  private
-
-  def unattached_media
-    MediaAttachment.reorder(nil).unattached.where('created_at < ?', 1.day.ago)
-  end
-end
diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb
new file mode 100644
index 00000000000..ce88ff204d3
--- /dev/null
+++ b/app/workers/scheduler/vacuum_scheduler.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Scheduler::VacuumScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 0
+
+  def perform
+    vacuum_operations.each do |operation|
+      operation.perform
+    rescue => e
+      Rails.logger.error("Error while running #{operation.class.name}: #{e}")
+    end
+  end
+
+  private
+
+  def vacuum_operations
+    [
+      statuses_vacuum,
+      media_attachments_vacuum,
+      preview_cards_vacuum,
+      backups_vacuum,
+      access_tokens_vacuum,
+      feeds_vacuum,
+    ]
+  end
+
+  def statuses_vacuum
+    Vacuum::StatusesVacuum.new(content_retention_policy.content_cache_retention_period)
+  end
+
+  def media_attachments_vacuum
+    Vacuum::MediaAttachmentsVacuum.new(content_retention_policy.media_cache_retention_period)
+  end
+
+  def preview_cards_vacuum
+    Vacuum::PreviewCardsVacuum.new(content_retention_policy.media_cache_retention_period)
+  end
+
+  def backups_vacuum
+    Vacuum::BackupsVacuum.new(content_retention_policy.backups_retention_period)
+  end
+
+  def access_tokens_vacuum
+    Vacuum::AccessTokensVacuum.new
+  end
+
+  def feeds_vacuum
+    Vacuum::FeedsVacuum.new
+  end
+
+  def content_retention_policy
+    ContentRetentionPolicy.current
+  end
+end
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index ddc83e8965c..db5b45e4103 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -73,6 +73,10 @@ en:
         actions:
           hide: Completely hide the filtered content, behaving as if it did not exist
           warn: Hide the filtered content behind a warning mentioning the filter's title
+      form_admin_settings:
+        backups_retention_period: Keep generated user archives for the specified number of days.
+        content_cache_retention_period: Posts from other servers will be deleted after the specified number of days when set to a positive value. This may be irreversible.
+        media_cache_retention_period: Downloaded media files will be deleted after the specified number of days when set to a positive value, and re-downloaded on demand.
       form_challenge:
         current_password: You are entering a secure area
       imports:
@@ -207,6 +211,10 @@ en:
         actions:
           hide: Hide completely
           warn: Hide with a warning
+      form_admin_settings:
+        backups_retention_period: User archive retention period
+        content_cache_retention_period: Content cache retention period
+        media_cache_retention_period: Media cache retention period
       interactions:
         must_be_follower: Block notifications from non-followers
         must_be_following: Block notifications from people you don't follow
diff --git a/config/settings.yml b/config/settings.yml
index eaa05071e11..41742118bc8 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -70,6 +70,7 @@ defaults: &defaults
   show_domain_blocks: 'disabled'
   show_domain_blocks_rationale: 'disabled'
   require_invite_text: false
+  backups_retention_period: 7
 
 development:
   <<: *defaults
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 9ec6eb5eca7..e3156aa346c 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -25,22 +25,14 @@
     every: '5m'
     class: Scheduler::IndexingScheduler
     queue: scheduler
-  media_cleanup_scheduler:
+  vacuum_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
-    class: Scheduler::MediaCleanupScheduler
-    queue: scheduler
-  feed_cleanup_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * *'
-    class: Scheduler::FeedCleanupScheduler
+    class: Scheduler::VacuumScheduler
     queue: scheduler
   follow_recommendations_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(6..9) %> * * *'
     class: Scheduler::FollowRecommendationsScheduler
     queue: scheduler
-  doorkeeper_cleanup_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * 0'
-    class: Scheduler::DoorkeeperCleanupScheduler
-    queue: scheduler
   user_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *'
     class: Scheduler::UserCleanupScheduler
@@ -49,10 +41,6 @@
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::IpCleanupScheduler
     queue: scheduler
-  backup_cleanup_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
-    class: Scheduler::BackupCleanupScheduler
-    queue: scheduler
   pghero_scheduler:
     cron: '0 0 * * *'
     class: Scheduler::PgheroScheduler
diff --git a/spec/fabricators/access_grant_fabricator.rb b/spec/fabricators/access_grant_fabricator.rb
new file mode 100644
index 00000000000..ae1945f2bb1
--- /dev/null
+++ b/spec/fabricators/access_grant_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator :access_grant, from: 'Doorkeeper::AccessGrant' do
+  application
+  resource_owner_id { Fabricate(:user).id }
+  expires_in 3_600
+  redirect_uri { Doorkeeper.configuration.native_redirect_uri }
+end
diff --git a/spec/fabricators/preview_card_fabricator.rb b/spec/fabricators/preview_card_fabricator.rb
index f119c117d98..99b5edc4357 100644
--- a/spec/fabricators/preview_card_fabricator.rb
+++ b/spec/fabricators/preview_card_fabricator.rb
@@ -3,4 +3,5 @@ Fabricator(:preview_card) do
   title { Faker::Lorem.sentence }
   description { Faker::Lorem.paragraph }
   type 'link'
+  image { attachment_fixture('attachment.jpg') }
 end
diff --git a/spec/lib/vacuum/access_tokens_vacuum_spec.rb b/spec/lib/vacuum/access_tokens_vacuum_spec.rb
new file mode 100644
index 00000000000..0244c344926
--- /dev/null
+++ b/spec/lib/vacuum/access_tokens_vacuum_spec.rb
@@ -0,0 +1,33 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::AccessTokensVacuum do
+  subject { described_class.new }
+
+  describe '#perform' do
+    let!(:revoked_access_token) { Fabricate(:access_token, revoked_at: 1.minute.ago) }
+    let!(:active_access_token) { Fabricate(:access_token) }
+
+    let!(:revoked_access_grant) { Fabricate(:access_grant, revoked_at: 1.minute.ago) }
+    let!(:active_access_grant) { Fabricate(:access_grant) }
+
+    before do
+      subject.perform
+    end
+
+    it 'deletes revoked access tokens' do
+      expect { revoked_access_token.reload }.to raise_error ActiveRecord::RecordNotFound
+    end
+
+    it 'deletes revoked access grants' do
+      expect { revoked_access_grant.reload }.to raise_error ActiveRecord::RecordNotFound
+    end
+
+    it 'does not delete active access tokens' do
+      expect { active_access_token.reload }.to_not raise_error
+    end
+
+    it 'does not delete active access grants' do
+      expect { active_access_grant.reload }.to_not raise_error
+    end
+  end
+end
diff --git a/spec/lib/vacuum/backups_vacuum_spec.rb b/spec/lib/vacuum/backups_vacuum_spec.rb
new file mode 100644
index 00000000000..4e2de083f8c
--- /dev/null
+++ b/spec/lib/vacuum/backups_vacuum_spec.rb
@@ -0,0 +1,24 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::BackupsVacuum do
+  let(:retention_period) { 7.days }
+
+  subject { described_class.new(retention_period) }
+
+  describe '#perform' do
+    let!(:expired_backup) { Fabricate(:backup, created_at: (retention_period + 1.day).ago) }
+    let!(:current_backup) { Fabricate(:backup) }
+
+    before do
+      subject.perform
+    end
+
+    it 'deletes backups past the retention period' do
+      expect { expired_backup.reload }.to raise_error ActiveRecord::RecordNotFound
+    end
+
+    it 'does not delete backups within the retention period' do
+      expect { current_backup.reload }.to_not raise_error
+    end
+  end
+end
diff --git a/spec/lib/vacuum/feeds_vacuum_spec.rb b/spec/lib/vacuum/feeds_vacuum_spec.rb
new file mode 100644
index 00000000000..0aec26740f3
--- /dev/null
+++ b/spec/lib/vacuum/feeds_vacuum_spec.rb
@@ -0,0 +1,30 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::FeedsVacuum do
+  subject { described_class.new }
+
+  describe '#perform' do
+    let!(:active_user) { Fabricate(:user, current_sign_in_at: 2.days.ago) }
+    let!(:inactive_user) { Fabricate(:user, current_sign_in_at: 22.days.ago) }
+
+    before do
+      redis.zadd(feed_key_for(inactive_user), 1, 1)
+      redis.zadd(feed_key_for(active_user), 1, 1)
+      redis.zadd(feed_key_for(inactive_user, 'reblogs'), 2, 2)
+      redis.sadd(feed_key_for(inactive_user, 'reblogs:2'), 3)
+
+      subject.perform
+    end
+
+    it 'clears feeds of inactive users and lists' do
+      expect(redis.zcard(feed_key_for(inactive_user))).to eq 0
+      expect(redis.zcard(feed_key_for(active_user))).to eq 1
+      expect(redis.exists?(feed_key_for(inactive_user, 'reblogs'))).to be false
+      expect(redis.exists?(feed_key_for(inactive_user, 'reblogs:2'))).to be false
+    end
+  end
+
+  def feed_key_for(user, subtype = nil)
+    FeedManager.instance.key(:home, user.account_id, subtype)
+  end
+end
diff --git a/spec/lib/vacuum/media_attachments_vacuum_spec.rb b/spec/lib/vacuum/media_attachments_vacuum_spec.rb
new file mode 100644
index 00000000000..be8458d9bf7
--- /dev/null
+++ b/spec/lib/vacuum/media_attachments_vacuum_spec.rb
@@ -0,0 +1,47 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::MediaAttachmentsVacuum do
+  let(:retention_period) { 7.days }
+
+  subject { described_class.new(retention_period) }
+
+  let(:remote_status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com')) }
+  let(:local_status) { Fabricate(:status) }
+
+  describe '#perform' do
+    let!(:old_remote_media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', status: remote_status, created_at: (retention_period + 1.day).ago, updated_at: (retention_period + 1.day).ago) }
+    let!(:old_local_media) { Fabricate(:media_attachment, status: local_status, created_at: (retention_period + 1.day).ago, updated_at: (retention_period + 1.day).ago) }
+    let!(:new_remote_media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', status: remote_status) }
+    let!(:new_local_media) { Fabricate(:media_attachment, status: local_status) }
+    let!(:old_unattached_media) { Fabricate(:media_attachment, account_id: nil, created_at: 10.days.ago) }
+    let!(:new_unattached_media) { Fabricate(:media_attachment, account_id: nil, created_at: 1.hour.ago) }
+
+    before do
+      subject.perform
+    end
+
+    it 'deletes cache of remote media attachments past the retention period' do
+      expect(old_remote_media.reload.file).to be_blank
+    end
+
+    it 'does not touch local media attachments past the retention period' do
+      expect(old_local_media.reload.file).to_not be_blank
+    end
+
+    it 'does not delete cache of remote media attachments within the retention period' do
+      expect(new_remote_media.reload.file).to_not be_blank
+    end
+
+    it 'does not touch local media attachments within the retention period' do
+      expect(new_local_media.reload.file).to_not be_blank
+    end
+
+    it 'deletes unattached media attachments past TTL' do
+      expect { old_unattached_media.reload }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+
+    it 'does not delete unattached media attachments within TTL' do
+      expect(new_unattached_media.reload).to be_persisted
+    end
+  end
+end
diff --git a/spec/lib/vacuum/preview_cards_vacuum_spec.rb b/spec/lib/vacuum/preview_cards_vacuum_spec.rb
new file mode 100644
index 00000000000..4a4a599fa47
--- /dev/null
+++ b/spec/lib/vacuum/preview_cards_vacuum_spec.rb
@@ -0,0 +1,36 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::PreviewCardsVacuum do
+  let(:retention_period) { 7.days }
+
+  subject { described_class.new(retention_period) }
+
+  describe '#perform' do
+    let!(:orphaned_preview_card) { Fabricate(:preview_card, created_at: 2.days.ago) }
+    let!(:old_preview_card) { Fabricate(:preview_card, updated_at: (retention_period + 1.day).ago) }
+    let!(:new_preview_card) { Fabricate(:preview_card) }
+
+    before do
+      old_preview_card.statuses << Fabricate(:status)
+      new_preview_card.statuses << Fabricate(:status)
+
+      subject.perform
+    end
+
+    it 'deletes cache of preview cards last updated before the retention period' do
+      expect(old_preview_card.reload.image).to be_blank
+    end
+
+    it 'does not delete cache of preview cards last updated within the retention period' do
+      expect(new_preview_card.reload.image).to_not be_blank
+    end
+
+    it 'does not delete attached preview cards' do
+      expect(new_preview_card.reload).to be_persisted
+    end
+
+    it 'deletes preview cards not attached to any status' do
+      expect { orphaned_preview_card.reload }.to raise_error ActiveRecord::RecordNotFound
+    end
+  end
+end
diff --git a/spec/lib/vacuum/statuses_vacuum_spec.rb b/spec/lib/vacuum/statuses_vacuum_spec.rb
new file mode 100644
index 00000000000..83f3c5c9f14
--- /dev/null
+++ b/spec/lib/vacuum/statuses_vacuum_spec.rb
@@ -0,0 +1,36 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::StatusesVacuum do
+  let(:retention_period) { 7.days }
+
+  let(:remote_account) { Fabricate(:account, domain: 'example.com') }
+
+  subject { described_class.new(retention_period) }
+
+  describe '#perform' do
+    let!(:remote_status_old) { Fabricate(:status, account: remote_account, created_at: (retention_period + 2.days).ago) }
+    let!(:remote_status_recent) { Fabricate(:status, account: remote_account, created_at: (retention_period - 2.days).ago) }
+    let!(:local_status_old) { Fabricate(:status, created_at: (retention_period + 2.days).ago) }
+    let!(:local_status_recent) { Fabricate(:status, created_at: (retention_period - 2.days).ago) }
+
+    before do
+      subject.perform
+    end
+
+    it 'deletes remote statuses past the retention period' do
+      expect { remote_status_old.reload }.to raise_error ActiveRecord::RecordNotFound
+    end
+
+    it 'does not delete local statuses past the retention period' do
+      expect { local_status_old.reload }.to_not raise_error
+    end
+
+    it 'does not delete remote statuses within the retention period' do
+      expect { remote_status_recent.reload }.to_not raise_error
+    end
+
+    it 'does not delete local statuses within the retention period' do
+      expect { local_status_recent.reload }.to_not raise_error
+    end
+  end
+end
diff --git a/spec/lib/vacuum/system_keys_vacuum_spec.rb b/spec/lib/vacuum/system_keys_vacuum_spec.rb
new file mode 100644
index 00000000000..565892f0252
--- /dev/null
+++ b/spec/lib/vacuum/system_keys_vacuum_spec.rb
@@ -0,0 +1,22 @@
+require 'rails_helper'
+
+RSpec.describe Vacuum::SystemKeysVacuum do
+  subject { described_class.new }
+
+  describe '#perform' do
+    let!(:expired_system_key) { Fabricate(:system_key, created_at: (SystemKey::ROTATION_PERIOD * 4).ago) }
+    let!(:current_system_key) { Fabricate(:system_key) }
+
+    before do
+      subject.perform
+    end
+
+    it 'deletes the expired key' do
+      expect { expired_system_key.reload }.to raise_error ActiveRecord::RecordNotFound
+    end
+
+    it 'does not delete the current key' do
+      expect { current_system_key.reload }.to_not raise_error
+    end
+  end
+end
diff --git a/spec/workers/scheduler/feed_cleanup_scheduler_spec.rb b/spec/workers/scheduler/feed_cleanup_scheduler_spec.rb
deleted file mode 100644
index 82d7945946c..00000000000
--- a/spec/workers/scheduler/feed_cleanup_scheduler_spec.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-require 'rails_helper'
-
-describe Scheduler::FeedCleanupScheduler do
-  subject { described_class.new }
-
-  let!(:active_user) { Fabricate(:user, current_sign_in_at: 2.days.ago) }
-  let!(:inactive_user) { Fabricate(:user, current_sign_in_at: 22.days.ago) }
-
-  it 'clears feeds of inactives' do
-    redis.zadd(feed_key_for(inactive_user), 1, 1)
-    redis.zadd(feed_key_for(active_user), 1, 1)
-    redis.zadd(feed_key_for(inactive_user, 'reblogs'), 2, 2)
-    redis.sadd(feed_key_for(inactive_user, 'reblogs:2'), 3)
-
-    subject.perform
-
-    expect(redis.zcard(feed_key_for(inactive_user))).to eq 0
-    expect(redis.zcard(feed_key_for(active_user))).to eq 1
-    expect(redis.exists?(feed_key_for(inactive_user, 'reblogs'))).to be false
-    expect(redis.exists?(feed_key_for(inactive_user, 'reblogs:2'))).to be false
-  end
-
-  def feed_key_for(user, subtype = nil)
-    FeedManager.instance.key(:home, user.account_id, subtype)
-  end
-end
diff --git a/spec/workers/scheduler/media_cleanup_scheduler_spec.rb b/spec/workers/scheduler/media_cleanup_scheduler_spec.rb
deleted file mode 100644
index 8a0da67e1b6..00000000000
--- a/spec/workers/scheduler/media_cleanup_scheduler_spec.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-require 'rails_helper'
-
-describe Scheduler::MediaCleanupScheduler do
-  subject { described_class.new }
-
-  let!(:old_media) { Fabricate(:media_attachment, account_id: nil, created_at: 10.days.ago) }
-  let!(:new_media) { Fabricate(:media_attachment, account_id: nil, created_at: 1.hour.ago) }
-
-  it 'removes old media records' do
-    subject.perform
-
-    expect { old_media.reload }.to raise_error(ActiveRecord::RecordNotFound)
-    expect(new_media.reload).to be_persisted
-  end
-end