diff --git a/app/controllers/api/fasp/data_sharing/v0/backfill_requests_controller.rb b/app/controllers/api/fasp/data_sharing/v0/backfill_requests_controller.rb index a5d5d8b791c..c37a94f2510 100644 --- a/app/controllers/api/fasp/data_sharing/v0/backfill_requests_controller.rb +++ b/app/controllers/api/fasp/data_sharing/v0/backfill_requests_controller.rb @@ -1,4 +1,26 @@ # frozen_string_literal: true -class Api::Fasp::DataSharing::V0::BackfillRequestsController < ApplicationController +class Api::Fasp::DataSharing::V0::BackfillRequestsController < Api::Fasp::BaseController + def create + backfill_request = current_provider.fasp_backfill_requests.new(backfill_request_params) + + respond_to do |format| + format.json do + if backfill_request.save + render json: { backfillRequest: { id: backfill_request.id } }, status: 201 + else + head 422 + end + end + end + end + + private + + def backfill_request_params + params + .permit(:category, :maxCount) + .to_unsafe_h + .transform_keys { |k| k.to_s.underscore } + end end diff --git a/app/controllers/api/fasp/data_sharing/v0/continuations_controller.rb b/app/controllers/api/fasp/data_sharing/v0/continuations_controller.rb new file mode 100644 index 00000000000..eff2ac0e213 --- /dev/null +++ b/app/controllers/api/fasp/data_sharing/v0/continuations_controller.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class Api::Fasp::DataSharing::V0::ContinuationsController < Api::Fasp::BaseController + def create + backfill_request = current_provider.fasp_backfill_requests.find(params[:backfill_request_id]) + Fasp::BackfillWorker.perform_async(backfill_request.id) + + head 204 + end +end diff --git a/app/models/fasp.rb b/app/models/fasp.rb index cb33937715c..e4e73a23127 100644 --- a/app/models/fasp.rb +++ b/app/models/fasp.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module Fasp + DATA_CATEGORIES = %w(account content).freeze + def self.table_name_prefix 'fasp_' end diff --git a/app/models/fasp/backfill_request.rb b/app/models/fasp/backfill_request.rb new file mode 100644 index 00000000000..e1be6110976 --- /dev/null +++ b/app/models/fasp/backfill_request.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: fasp_backfill_requests +# +# id :bigint(8) not null, primary key +# category :string not null +# cursor :string +# fulfilled :boolean default(FALSE), not null +# max_count :integer default(100), not null +# created_at :datetime not null +# updated_at :datetime not null +# fasp_provider_id :bigint(8) not null +# +class Fasp::BackfillRequest < ApplicationRecord + belongs_to :fasp_provider, class_name: 'Fasp::Provider' + + validates :category, presence: true, inclusion: Fasp::DATA_CATEGORIES + validates :max_count, presence: true, + numericality: { only_integer: true } + + after_commit :queue_fulfillment_job, on: :create + + def next_objects + @next_objects ||= base_scope.to_a + end + + def next_uris + next_objects.map { |o| ActivityPub::TagManager.instance.uri_for(o) } + end + + def more_objects_available? + return false if next_objects.empty? + + base_scope.where(id: ...(next_objects.last.id)).any? + end + + def advance! + if more_objects_available? + update!(cursor: next_objects.last.id) + else + update!(fulfilled: true) + end + end + + private + + def base_scope + result = category_scope.limit(max_count).order(id: :desc) + result = result.where(id: ...cursor) if cursor.present? + result + end + + def category_scope + case category + when 'account' + Account.discoverable.without_instance_actor + when 'content' + Status.indexable + end + end + + def queue_fulfillment_job + Fasp::BackfillWorker.perform_async(id) + end +end diff --git a/app/models/fasp/provider.rb b/app/models/fasp/provider.rb index eff4e826c58..5a2057e9298 100644 --- a/app/models/fasp/provider.rb +++ b/app/models/fasp/provider.rb @@ -22,6 +22,7 @@ class Fasp::Provider < ApplicationRecord include DebugConcern + has_many :fasp_backfill_requests, inverse_of: :fasp_provider, class_name: 'Fasp::BackfillRequest', dependent: :delete_all has_many :fasp_debug_callbacks, inverse_of: :fasp_provider, class_name: 'Fasp::DebugCallback', dependent: :delete_all has_many :fasp_subscriptions, inverse_of: :fasp_provider, class_name: 'Fasp::Subscription', dependent: :delete_all diff --git a/app/models/fasp/subscription.rb b/app/models/fasp/subscription.rb index 15236134507..80e8ed691c0 100644 --- a/app/models/fasp/subscription.rb +++ b/app/models/fasp/subscription.rb @@ -17,12 +17,11 @@ # fasp_provider_id :bigint(8) not null # class Fasp::Subscription < ApplicationRecord - CATEGORIES = %w(account content).freeze TYPES = %w(lifecycle trends).freeze belongs_to :fasp_provider, class_name: 'Fasp::Provider' - validates :category, presence: true, inclusion: CATEGORIES + validates :category, presence: true, inclusion: Fasp::DATA_CATEGORIES validates :subscription_type, presence: true, inclusion: TYPES diff --git a/app/workers/fasp/backfill_worker.rb b/app/workers/fasp/backfill_worker.rb new file mode 100644 index 00000000000..4e30b71a7dd --- /dev/null +++ b/app/workers/fasp/backfill_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class Fasp::BackfillWorker + include Sidekiq::Worker + + sidekiq_options queue: 'fasp', retry: 5 + + def perform(backfill_request_id) + backfill_request = Fasp::BackfillRequest.find(backfill_request_id) + + announce(backfill_request) + + backfill_request.advance! + rescue ActiveRecord::RecordNotFound + # ignore missing backfill requests + end + + private + + def announce(backfill_request) + Fasp::Request.new(backfill_request.fasp_provider).post('/data_sharing/v0/announcements', body: { + source: { + backfillRequest: { + id: backfill_request.id.to_s, + }, + }, + category: backfill_request.category, + objectUris: backfill_request.next_uris, + moreObjectsAvailable: backfill_request.more_objects_available?, + }) + end +end diff --git a/config/routes/fasp.rb b/config/routes/fasp.rb index cb9b8942903..bd2bb4b5208 100644 --- a/config/routes/fasp.rb +++ b/config/routes/fasp.rb @@ -12,7 +12,9 @@ namespace :api, format: false do namespace :data_sharing do namespace :v0 do - resources :backfill_requests, only: [:create] + resources :backfill_requests, only: [:create] do + resource :continuation, only: [:create] + end resources :event_subscriptions, only: [:create, :destroy] end diff --git a/db/migrate/20250103131909_create_fasp_backfill_requests.rb b/db/migrate/20250103131909_create_fasp_backfill_requests.rb new file mode 100644 index 00000000000..31dcaaa469a --- /dev/null +++ b/db/migrate/20250103131909_create_fasp_backfill_requests.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateFaspBackfillRequests < ActiveRecord::Migration[7.2] + def change + create_table :fasp_backfill_requests do |t| + t.string :category, null: false + t.integer :max_count, null: false, default: 100 + t.string :cursor + t.boolean :fulfilled, null: false, default: false + t.references :fasp_provider, null: false, foreign_key: true + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 0353efcd26c..671f9b75ba2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,9 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_12_16_224825) do +ActiveRecord::Schema[8.0].define(version: 2025_01_03_131909) do # These are extensions that must be enabled in order to support this database - enable_extension "plpgsql" + enable_extension "pg_catalog.plpgsql" create_table "account_aliases", force: :cascade do |t| t.bigint "account_id", null: false @@ -444,6 +444,17 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_16_224825) do t.index ["domain"], name: "index_email_domain_blocks_on_domain", unique: true end + create_table "fasp_backfill_requests", force: :cascade do |t| + t.string "category", null: false + t.integer "max_count", default: 100, null: false + t.string "cursor" + t.boolean "fulfilled", default: false, null: false + t.bigint "fasp_provider_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["fasp_provider_id"], name: "index_fasp_backfill_requests_on_fasp_provider_id" + end + create_table "fasp_debug_callbacks", force: :cascade do |t| t.bigint "fasp_provider_id", null: false t.string "ip" @@ -1323,6 +1334,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_16_224825) do add_foreign_key "custom_filter_statuses", "statuses", on_delete: :cascade add_foreign_key "custom_filters", "accounts", on_delete: :cascade add_foreign_key "email_domain_blocks", "email_domain_blocks", column: "parent_id", on_delete: :cascade + add_foreign_key "fasp_backfill_requests", "fasp_providers" add_foreign_key "fasp_debug_callbacks", "fasp_providers" add_foreign_key "fasp_subscriptions", "fasp_providers" add_foreign_key "favourites", "accounts", name: "fk_5eb6c2b873", on_delete: :cascade diff --git a/spec/fabricators/fasp/backfill_request_fabricator.rb b/spec/fabricators/fasp/backfill_request_fabricator.rb new file mode 100644 index 00000000000..fc9461be275 --- /dev/null +++ b/spec/fabricators/fasp/backfill_request_fabricator.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +Fabricator('Fasp::BackfillRequest') do + category 'MyString' + max_count 1 + cursor 'MyString' + fulfilled false + fasp_provider nil +end diff --git a/spec/models/fasp/backfill_request_spec.rb b/spec/models/fasp/backfill_request_spec.rb new file mode 100644 index 00000000000..397a9e942b8 --- /dev/null +++ b/spec/models/fasp/backfill_request_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Fasp::BackfillRequest do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/requests/api/fasp/data_sharing/v0/continuations_spec.rb b/spec/requests/api/fasp/data_sharing/v0/continuations_spec.rb new file mode 100644 index 00000000000..67265fc96d8 --- /dev/null +++ b/spec/requests/api/fasp/data_sharing/v0/continuations_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'Api::Fasp::DataSharing::V0::Continuations' do + describe 'GET /index' do + pending "add some examples (or delete) #{__FILE__}" + end +end