Introduce Ostatus name space (#4164)

* Wrap methods of ProcessFeedService::ProcessEntry in classes

This is a change same with 425acecfdb, except
that it has the following changes:

* Revert irrelevant change in find_or_create_conversation
* Fix error handling for RemoteActivity

* Introduce Ostatus name space
This commit is contained in:
Akihiko Odaki 2017-07-18 23:39:47 +09:00 committed by Eugen Rochko
parent 4d42a38954
commit 89b988cab5
23 changed files with 516 additions and 447 deletions

View File

@ -13,7 +13,7 @@ class AccountsController < ApplicationController
format.atom do format.atom do
@entries = @account.stream_entries.where(hidden: false).with_includes.paginate_by_max_id(20, params[:max_id], params[:since_id]) @entries = @account.stream_entries.where(hidden: false).with_includes.paginate_by_max_id(20, params[:max_id], params[:since_id])
render xml: AtomSerializer.render(AtomSerializer.new.feed(@account, @entries.to_a)) render xml: Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.feed(@account, @entries.to_a))
end end
format.json do format.json do

View File

@ -19,7 +19,7 @@ class StreamEntriesController < ApplicationController
end end
format.atom do format.atom do
render xml: AtomSerializer.render(AtomSerializer.new.entry(@stream_entry, true)) render xml: Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.entry(@stream_entry, true))
end end
end end
end end

View File

@ -0,0 +1,50 @@
# frozen_string_literal: true
class Ostatus::Activity::Base
def initialize(xml, account = nil)
@xml = xml
@account = account
end
def status?
[:activity, :note, :comment].include?(type)
end
def verb
raw = @xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content
TagManager::VERBS.key(raw)
rescue
:post
end
def type
raw = @xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content
TagManager::TYPES.key(raw)
rescue
:activity
end
def id
@xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content
end
def url
link = @xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS)
link.nil? ? nil : link['href']
end
private
def find_status(uri)
if TagManager.instance.local_id?(uri)
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status')
return Status.find_by(id: local_id)
end
Status.find_by(uri: uri)
end
def redis
Redis.current
end
end

View File

@ -0,0 +1,149 @@
# frozen_string_literal: true
class Ostatus::Activity::Creation < Ostatus::Activity::Base
def perform
if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
Rails.logger.debug "Delete for status #{id} was queued, ignoring"
return [nil, false]
end
return [nil, false] if @account.suspended?
Rails.logger.debug "Creating remote status #{id}"
# Return early if status already exists in db
status = find_status(id)
return [status, false] unless status.nil?
status = Status.create!(
uri: id,
url: url,
account: @account,
reblog: reblog,
text: content,
spoiler_text: content_warning,
created_at: published,
reply: thread?,
language: content_language,
visibility: visibility_scope,
conversation: find_or_create_conversation,
thread: thread? ? find_status(thread.first) : nil
)
save_mentions(status)
save_hashtags(status)
save_media(status)
if thread? && status.thread.nil?
Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
ThreadResolveWorker.perform_async(status.id, thread.second)
end
Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
DistributionWorker.perform_async(status.id)
[status, true]
end
def content
@xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content
end
def content_language
@xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en'
end
def content_warning
@xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || ''
end
def visibility_scope
@xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public
end
def published
@xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content
end
def thread?
!@xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil?
end
def thread
thr = @xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS)
[thr['ref'], thr['href']]
end
private
def find_or_create_conversation
uri = @xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content
return if uri.nil?
if TagManager.instance.local_id?(uri)
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
return Conversation.find_by(id: local_id)
end
Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
end
def save_mentions(parent)
processed_account_ids = []
@xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link|
next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type']
mentioned_account = account_from_href(link['href'])
next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
# So we can skip duplicate mentions
processed_account_ids << mentioned_account.id
end
end
def save_hashtags(parent)
tags = @xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
ProcessHashtagsService.new.call(parent, tags)
end
def save_media(parent)
do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
@xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link|
next unless link['href']
media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
parsed_url = Addressable::URI.parse(link['href']).normalize
next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
media.save
next if do_not_download
begin
media.file_remote_url = link['href']
media.save!
rescue ActiveRecord::RecordInvalid
next
end
end
end
def account_from_href(href)
url = Addressable::URI.parse(href).normalize
if TagManager.instance.web_domain?(url.host)
Account.find_local(url.path.gsub('/users/', ''))
else
Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
end
end
end

View File

@ -0,0 +1,14 @@
# frozen_string_literal: true
class Ostatus::Activity::Deletion < Ostatus::Activity::Base
def perform
Rails.logger.debug "Deleting remote status #{id}"
status = Status.find_by(uri: id, account: @account)
if status.nil?
redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
else
RemoveStatusService.new.call(status)
end
end
end

View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
class Ostatus::Activity::General < Ostatus::Activity::Base
def specialize
special_class&.new(@xml, @account)
end
private
def special_class
case verb
when :post
Ostatus::Activity::Post
when :share
Ostatus::Activity::Share
when :delete
Ostatus::Activity::Deletion
end
end
end

View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
class Ostatus::Activity::Post < Ostatus::Activity::Creation
def perform
status, just_created = super
if just_created
status.mentions.includes(:account).each do |mention|
mentioned_account = mention.account
next unless mentioned_account.local?
NotifyService.new.call(mentioned_account, mention)
end
end
status
end
private
def reblog
nil
end
end

View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
class Ostatus::Activity::Remote < Ostatus::Activity::Base
def perform
find_status(id) || FetchRemoteStatusService.new.call(url)
end
end

View File

@ -0,0 +1,26 @@
# frozen_string_literal: true
class Ostatus::Activity::Share < Ostatus::Activity::Creation
def perform
return if reblog.nil?
status, just_created = super
NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created
status
end
def object
@xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS)
end
private
def reblog
return @reblog if defined? @reblog
original_status = Ostatus::Activity::Remote.new(object).perform
return if original_status.nil?
@reblog = original_status.reblog? ? original_status.reblog : original_status
end
end

View File

@ -1,6 +1,6 @@
# frozen_string_literal: true # frozen_string_literal: true
class AtomSerializer class Ostatus::AtomSerializer
include RoutingHelper include RoutingHelper
include ActionView::Helpers::SanitizeHelper include ActionView::Helpers::SanitizeHelper

View File

@ -10,6 +10,6 @@ class AuthorizeFollowService < BaseService
private private
def build_xml(follow_request) def build_xml(follow_request)
AtomSerializer.render(AtomSerializer.new.authorize_follow_request_salmon(follow_request)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request))
end end
end end

View File

@ -18,6 +18,6 @@ class BlockService < BaseService
private private
def build_xml(block) def build_xml(block)
AtomSerializer.render(AtomSerializer.new.block_salmon(block)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.block_salmon(block))
end end
end end

View File

@ -2,6 +2,6 @@
module StreamEntryRenderer module StreamEntryRenderer
def stream_entry_to_xml(stream_entry) def stream_entry_to_xml(stream_entry)
AtomSerializer.render(AtomSerializer.new.entry(stream_entry, true)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.entry(stream_entry, true))
end end
end end

View File

@ -28,6 +28,6 @@ class FavouriteService < BaseService
private private
def build_xml(favourite) def build_xml(favourite)
AtomSerializer.render(AtomSerializer.new.favourite_salmon(favourite)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.favourite_salmon(favourite))
end end
end end

View File

@ -57,10 +57,10 @@ class FollowService < BaseService
end end
def build_follow_request_xml(follow_request) def build_follow_request_xml(follow_request)
AtomSerializer.render(AtomSerializer.new.follow_request_salmon(follow_request)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.follow_request_salmon(follow_request))
end end
def build_follow_xml(follow) def build_follow_xml(follow)
AtomSerializer.render(AtomSerializer.new.follow_salmon(follow)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.follow_salmon(follow))
end end
end end

View File

@ -16,274 +16,14 @@ class ProcessFeedService < BaseService
end end
def process_entries(xml, account) def process_entries(xml, account)
xml.xpath('//xmlns:entry', xmlns: TagManager::XMLNS).reverse_each.map { |entry| ProcessEntry.new.call(entry, account) }.compact xml.xpath('//xmlns:entry', xmlns: TagManager::XMLNS).reverse_each.map { |entry| process_entry(entry, account) }.compact
end end
class ProcessEntry def process_entry(xml, account)
def call(xml, account) activity = Ostatus::Activity::General.new(xml, account)
@account = account activity.specialize&.perform if activity.status?
@xml = xml
return if skip_unsupported_type?
case verb
when :post, :share
return create_status
when :delete
return delete_status
end
rescue ActiveRecord::RecordInvalid => e rescue ActiveRecord::RecordInvalid => e
Rails.logger.debug "Nothing was saved for #{id} because: #{e}" Rails.logger.debug "Nothing was saved for #{id} because: #{e}"
nil nil
end end
private
def create_status
if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
Rails.logger.debug "Delete for status #{id} was queued, ignoring"
return
end
status, just_created = nil
Rails.logger.debug "Creating remote status #{id}"
if verb == :share
original_status = shared_status_from_xml(@xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS))
return nil if original_status.nil?
end
ApplicationRecord.transaction do
status, just_created = status_from_xml(@xml)
return if status.nil?
return status unless just_created
if verb == :share
status.reblog = original_status.reblog? ? original_status.reblog : original_status
end
status.save!
end
if thread?(@xml) && status.thread.nil?
Rails.logger.debug "Trying to attach #{status.id} (#{id(@xml)}) to #{thread(@xml).first}"
ThreadResolveWorker.perform_async(status.id, thread(@xml).second)
end
notify_about_mentions!(status) unless status.reblog?
notify_about_reblog!(status) if status.reblog? && status.reblog.account.local?
Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
DistributionWorker.perform_async(status.id)
status
end
def notify_about_mentions!(status)
status.mentions.includes(:account).each do |mention|
mentioned_account = mention.account
next unless mentioned_account.local?
NotifyService.new.call(mentioned_account, mention)
end
end
def notify_about_reblog!(status)
NotifyService.new.call(status.reblog.account, status)
end
def delete_status
Rails.logger.debug "Deleting remote status #{id}"
status = Status.find_by(uri: id, account: @account)
if status.nil?
redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
else
RemoveStatusService.new.call(status)
end
end
def skip_unsupported_type?
!([:post, :share, :delete].include?(verb) && [:activity, :note, :comment].include?(type))
end
def shared_status_from_xml(entry)
status = find_status(id(entry))
return status unless status.nil?
FetchRemoteStatusService.new.call(url(entry))
end
def status_from_xml(entry)
# Return early if status already exists in db
status = find_status(id(entry))
return [status, false] unless status.nil?
account = @account
return [nil, false] if account.suspended?
status = Status.create!(
uri: id(entry),
url: url(entry),
account: account,
text: content(entry),
spoiler_text: content_warning(entry),
created_at: published(entry),
reply: thread?(entry),
language: content_language(entry),
visibility: visibility_scope(entry),
conversation: find_or_create_conversation(entry),
thread: thread?(entry) ? find_status(thread(entry).first) : nil
)
mentions_from_xml(status, entry)
hashtags_from_xml(status, entry)
media_from_xml(status, entry)
[status, true]
end
def find_or_create_conversation(xml)
uri = xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content
return if uri.nil?
if TagManager.instance.local_id?(uri)
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
return Conversation.find_by(id: local_id)
end
Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
end
def find_status(uri)
if TagManager.instance.local_id?(uri)
local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status')
return Status.find_by(id: local_id)
end
Status.find_by(uri: uri)
end
def mentions_from_xml(parent, xml)
processed_account_ids = []
xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link|
next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type']
mentioned_account = account_from_href(link['href'])
next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
# So we can skip duplicate mentions
processed_account_ids << mentioned_account.id
end
end
def account_from_href(href)
url = Addressable::URI.parse(href).normalize
if TagManager.instance.web_domain?(url.host)
Account.find_local(url.path.gsub('/users/', ''))
else
Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
end
end
def hashtags_from_xml(parent, xml)
tags = xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
ProcessHashtagsService.new.call(parent, tags)
end
def media_from_xml(parent, xml)
do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link|
next unless link['href']
media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
parsed_url = Addressable::URI.parse(link['href']).normalize
next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
media.save
next if do_not_download
begin
media.file_remote_url = link['href']
media.save!
rescue ActiveRecord::RecordInvalid
next
end
end
end
def id(xml = @xml)
xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content
end
def verb(xml = @xml)
raw = xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content
TagManager::VERBS.key(raw)
rescue
:post
end
def type(xml = @xml)
raw = xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content
TagManager::TYPES.key(raw)
rescue
:activity
end
def url(xml = @xml)
link = xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS)
link.nil? ? nil : link['href']
end
def content(xml = @xml)
xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content
end
def content_language(xml = @xml)
xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en'
end
def content_warning(xml = @xml)
xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || ''
end
def visibility_scope(xml = @xml)
xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public
end
def published(xml = @xml)
xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content
end
def thread?(xml = @xml)
!xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil?
end
def thread(xml = @xml)
thr = xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS)
[thr['ref'], thr['href']]
end
def account?(xml = @xml)
!xml.at_xpath('./xmlns:author', xmlns: TagManager::XMLNS).nil?
end
def redis
Redis.current
end
end
end end

View File

@ -10,6 +10,6 @@ class RejectFollowService < BaseService
private private
def build_xml(follow_request) def build_xml(follow_request)
AtomSerializer.render(AtomSerializer.new.reject_follow_request_salmon(follow_request)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.reject_follow_request_salmon(follow_request))
end end
end end

View File

@ -11,6 +11,6 @@ class UnblockService < BaseService
private private
def build_xml(block) def build_xml(block)
AtomSerializer.render(AtomSerializer.new.unblock_salmon(block)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.unblock_salmon(block))
end end
end end

View File

@ -13,6 +13,6 @@ class UnfavouriteService < BaseService
private private
def build_xml(favourite) def build_xml(favourite)
AtomSerializer.render(AtomSerializer.new.unfavourite_salmon(favourite)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.unfavourite_salmon(favourite))
end end
end end

View File

@ -14,6 +14,6 @@ class UnfollowService < BaseService
private private
def build_xml(follow) def build_xml(follow)
AtomSerializer.render(AtomSerializer.new.unfollow_salmon(follow)) Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.unfollow_salmon(follow))
end end
end end

View File

@ -22,7 +22,7 @@ class Pubsubhubbub::DistributionWorker
def distribute_public!(stream_entries) def distribute_public!(stream_entries)
return if stream_entries.empty? return if stream_entries.empty?
@payload = AtomSerializer.render(AtomSerializer.new.feed(@account, stream_entries)) @payload = Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.feed(@account, stream_entries))
Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription| Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription|
[subscription.id, @payload] [subscription.id, @payload]
@ -32,7 +32,7 @@ class Pubsubhubbub::DistributionWorker
def distribute_hidden!(stream_entries) def distribute_hidden!(stream_entries)
return if stream_entries.empty? return if stream_entries.empty?
@payload = AtomSerializer.render(AtomSerializer.new.feed(@account, stream_entries)) @payload = Ostatus::AtomSerializer.render(Ostatus::AtomSerializer.new.feed(@account, stream_entries))
@domains = @account.followers.domains @domains = @account.followers.domains
Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions.reject { |s| !allowed_to_receive?(s.callback_url, s.domain) }) do |subscription| Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions.reject { |s| !allowed_to_receive?(s.callback_url, s.domain) }) do |subscription|

View File

@ -167,6 +167,46 @@ XML
expect(created_statuses.first.reblog.text).to eq 'Overwatch rocks' expect(created_statuses.first.reblog.text).to eq 'Overwatch rocks'
end end
it 'ignores reblogs if it failed to retreive reblogged statuses' do
stub_request(:head, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404)
actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
body = <<XML
<?xml version="1.0"?>
<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
<id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
<published>2017-04-27T13:49:25Z</published>
<updated>2017-04-27T13:49:25Z</updated>
<author>
<id>https://overwatch.com/users/tracer</id>
<activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
<uri>https://overwatch.com/users/tracer</uri>
<name>tracer</name>
</author>
<activity:object-type>http://activitystrea.ms/schema/1.0/activity</activity:object-type>
<activity:verb>http://activitystrea.ms/schema/1.0/share</activity:verb>
<content type="html">Overwatch rocks</content>
<activity:object>
<id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
<activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
<activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
<author>
<id>https://overwatch.com/users/tracer</id>
<activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
<uri>https://overwatch.com/users/tracer</uri>
<name>tracer</name>
</author>
<content type="html">Overwatch rocks</content>
<link rel="alternate" type="text/html" href="https://overwatch.com/users/tracer/updates/1" />
</activity:object>
XML
created_statuses = subject.call(body, actor)
expect(created_statuses).to eq []
end
it 'ignores statuses with an out-of-order delete' do it 'ignores statuses with an out-of-order delete' do
sender = Fabricate(:account, username: 'tracer', domain: 'overwatch.com') sender = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')