rubiline
/
Glitch-Shrike
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add SELF_DESTRUCT env variable to process self-destructions in the background (#26439)

shrike
Claire 2023-10-23 17:46:21 +02:00 committed by GitHub
parent 26d2a2a0cc
commit 379115e601
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 193 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/controllers/application_controller.rb
View File

@ -12,6 +12,7 @@ class ApplicationController < ActionController::Base
include DomainControlHelper
include DatabaseHelper
include AuthorizedFetchHelper
include SelfDestructHelper
helper_method :current_account
helper_method :current_session
@ -39,6 +40,8 @@ class ApplicationController < ActionController::Base
service_unavailable
end
before_action :check_self_destruct!
before_action :store_referrer, except: :raise_not_found, if: :devise_controller?
before_action :require_functional!, if: :user_signed_in?
@ -170,6 +173,15 @@ class ApplicationController < ActionController::Base
end
end
def check_self_destruct!
return unless self_destruct?
respond_to do |format|
format.any { render 'errors/self_destruct', layout: 'auth', status: 410, formats: [:html] }
format.json { render json: { error: Rack::Utils::HTTP_STATUS_CODES[410] }, status: code }
end
end
def set_cache_control_defaults
response.cache_control.replace(private: true, no_store: true)
end

1
app/controllers/auth/challenges_controller.rb
View File

@ -7,6 +7,7 @@ class Auth::ChallengesController < ApplicationController
before_action :authenticate_user!
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
def create

1
app/controllers/auth/confirmations_controller.rb
View File

@ -12,6 +12,7 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController
before_action :extend_csp_for_captcha!, only: [:show, :confirm_captcha]
before_action :require_captcha_if_needed!, only: [:show]
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
def show

1
app/controllers/auth/omniauth_callbacks_controller.rb
View File

@ -1,6 +1,7 @@
# frozen_string_literal: true
class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
skip_before_action :check_self_destruct!
skip_before_action :verify_authenticity_token
def self.provides_callback_for(provider)

1
app/controllers/auth/passwords_controller.rb
View File

@ -1,6 +1,7 @@
# frozen_string_literal: true
class Auth::PasswordsController < Devise::PasswordsController
skip_before_action :check_self_destruct!
before_action :check_validity_of_reset_password_token, only: :edit
before_action :set_body_classes

1
app/controllers/auth/registrations_controller.rb
View File

@ -17,6 +17,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
before_action :require_rules_acceptance!, only: :new
before_action :set_registration_form_time, only: :new
skip_before_action :check_self_destruct!, only: [:edit, :update]
skip_before_action :require_functional!, only: [:edit, :update]
def new

1
app/controllers/auth/sessions_controller.rb
View File

@ -3,6 +3,7 @@
class Auth::SessionsController < Devise::SessionsController
layout 'auth'
skip_before_action :check_self_destruct!
skip_before_action :require_no_authentication, only: [:create]
skip_before_action :require_functional!
skip_before_action :update_user_sign_in

1
app/controllers/backups_controller.rb
View File

@ -3,6 +3,7 @@
class BackupsController < ApplicationController
include RoutingHelper
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
before_action :authenticate_user!

1
app/controllers/concerns/export_controller_concern.rb
View File

@ -7,6 +7,7 @@ module ExportControllerConcern
before_action :authenticate_user!
before_action :load_export
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
end

1
app/controllers/settings/exports_controller.rb
View File

@ -5,6 +5,7 @@ class Settings::ExportsController < Settings::BaseController
include Redisable
include Lockable
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
def show

3
app/controllers/settings/login_activities_controller.rb
View File

@ -1,6 +1,9 @@
# frozen_string_literal: true
class Settings::LoginActivitiesController < Settings::BaseController
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
def index
@login_activities = LoginActivity.where(user: current_user).order(id: :desc).page(params[:page])
end

1
app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb
View File

@ -3,6 +3,7 @@
module Settings
module TwoFactorAuthentication
class WebauthnCredentialsController < BaseController
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
before_action :require_otp_enabled

1
app/controllers/settings/two_factor_authentication_methods_controller.rb
View File

@ -4,6 +4,7 @@ module Settings
class TwoFactorAuthenticationMethodsController < BaseController
include ChallengableConcern
skip_before_action :check_self_destruct!
skip_before_action :require_functional!
before_action :require_challenge!, only: :disable

14
app/helpers/self_destruct_helper.rb Normal file
View File

@ -0,0 +1,14 @@
# frozen_string_literal: true
module SelfDestructHelper
def self.self_destruct?
value = ENV.fetch('SELF_DESTRUCT', nil)
value.present? && Rails.application.message_verifier('self-destruct').verify(value) == ENV['LOCAL_DOMAIN']
rescue ActiveSupport::MessageVerifier::InvalidSignature
false
end
def self_destruct?
SelfDestructHelper.self_destruct?
end
end

6
app/views/auth/registrations/edit.html.haml
View File

@ -1,6 +1,10 @@
- content_for :page_title do
= t('settings.account_settings')
- if self_destruct?
.flash-message.warning
= t('auth.status.self_destruct', domain: ENV['LOCAL_DOMAIN'])
- else
= render partial: 'status', locals: { user: @user, strikes: @strikes }
%h3= t('auth.security')
@ -32,7 +36,7 @@
= render partial: 'sessions', object: @sessions
- unless current_account.suspended?
- unless current_account.suspended? || self_destruct?
%hr.spacer/
%h3= t('auth.migrate_account')

20
app/views/errors/self_destruct.html.haml Normal file
View File

@ -0,0 +1,20 @@
- content_for :page_title do
= t('self_destruct.title')
.simple_form
%h1.title= t('self_destruct.title')
%p.lead= t('self_destruct.lead_html', domain: ENV['LOCAL_DOMAIN'])
.form-footer
%ul.no-list
- if user_signed_in?
%li= link_to t('settings.account_settings'), edit_user_registration_path
- else
- if controller_name != 'sessions'
%li= link_to_login t('auth.login')
- if controller_name != 'passwords' && controller_name != 'registrations'
%li= link_to t('auth.forgot_password'), new_user_password_path
- if user_signed_in?
%li= link_to t('auth.logout'), destroy_user_session_path, data: { method: :delete }

72
app/workers/scheduler/self_destruct_scheduler.rb Normal file
View File

@ -0,0 +1,72 @@
# frozen_string_literal: true
class Scheduler::SelfDestructScheduler
include Sidekiq::Worker
include SelfDestructHelper
MAX_ENQUEUED = 10_000
MAX_REDIS_MEM_USAGE = 0.5
MAX_ACCOUNT_DELETIONS_PER_JOB = 50
sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
def perform
return unless self_destruct?
return if sidekiq_overwhelmed?
delete_accounts!
end
private
def sidekiq_overwhelmed?
redis_mem_info = Sidekiq.redis_info
Sidekiq::Stats.new.enqueued > MAX_ENQUEUED || redis_mem_info['used_memory'].to_f > redis_mem_info['total_system_memory'].to_f * MAX_REDIS_MEM_USAGE
end
def delete_accounts!
# We currently do not distinguish between deleted accounts and suspended
# accounts, and we do not want to remove the records in this scheduler, as
# we still rely on it for account delivery and don't want to perform
# needless work when the database can be outright dropped after the
# self-destruct.
# Deleted accounts are suspended accounts that do not have a pending
# deletion request.
# This targets accounts that have not been deleted nor marked for deletion yet
Account.local.without_suspended.reorder(id: :asc).take(MAX_ACCOUNT_DELETIONS_PER_JOB).each do |account|
delete_account!(account)
end
return if sidekiq_overwhelmed?
# This targets accounts that have been marked for deletion but have not been
# deleted yet
Account.local.suspended.joins(:deletion_request).take(MAX_ACCOUNT_DELETIONS_PER_JOB).each do |account|
delete_account!(account)
account.deletion_request&.destroy
end
end
def inboxes
@inboxes ||= Account.inboxes
end
def delete_account!(account)
payload = ActiveModelSerializers::SerializableResource.new(
account,
serializer: ActivityPub::DeleteActorSerializer,
adapter: ActivityPub::Adapter
).as_json
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
[json, account.id, inbox_url]
end
# Do not call `Account#suspend!` because we don't want to issue a deletion request
account.update!(suspended_at: Time.now.utc, suspension_origin: :local)
end
end

12
config/initializers/sidekiq.rb
View File

@ -17,6 +17,18 @@ Sidekiq.configure_server do |config|
chain.add SidekiqUniqueJobs::Middleware::Client
end
config.on(:startup) do
if SelfDestructHelper.self_destruct?
Sidekiq.schedule = {
'self_destruct_scheduler' => {
'interval' => ['1m'],
'class' => 'Scheduler::SelfDestructScheduler',
'queue' => 'scheduler',
},
}
end
end
SidekiqUniqueJobs::Server.configure(config)
end

4
config/locales/en.yml
View File

@ -1102,6 +1102,7 @@ en:
functional: Your account is fully operational.
pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
redirecting_to: Your account is inactive because it is currently redirecting to %{acct}.
self_destruct: As %{domain} is closing down, you will only get limited access to your account.
view_strikes: View past strikes against your account
too_fast: Form submitted too fast, try again.
use_security_key: Use security key
@ -1572,6 +1573,9 @@ en:
over_daily_limit: You have exceeded the limit of %{limit} scheduled posts for today
over_total_limit: You have exceeded the limit of %{limit} scheduled posts
too_soon: The scheduled date must be in the future
self_destruct:
lead_html: Unfortunately, <strong>%{domain}</strong> is permanently closing down. If you had an account there, you will not be able to continue using it, but you can still request a backup of your data.
title: This server is closing down
sessions:
activity: Last activity
browser: Browser

26
config/navigation.rb
View File

@ -1,44 +1,46 @@
# frozen_string_literal: true
SimpleNavigation::Configuration.run do |navigation|
self_destruct = SelfDestructHelper.self_destruct?
navigation.items do |n|
n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_path
n.item :software_updates, safe_join([fa_icon('exclamation-circle fw'), t('admin.critical_update_pending')]), admin_software_updates_path, if: -> { ENV['UPDATE_CHECK_URL'] != '' && current_user.can?(:view_devops) && SoftwareUpdate.urgent_pending? }, html: { class: 'warning' }
n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_path, if: -> { current_user.functional? }, highlights_on: %r{/settings/profile|/settings/featured_tags|/settings/verification|/settings/privacy}
n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_path, if: -> { current_user.functional? && !self_destruct }, highlights_on: %r{/settings/profile|/settings/featured_tags|/settings/verification|/settings/privacy}
n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_path, if: -> { current_user.functional? } do |s|
n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_path, if: -> { current_user.functional? && !self_destruct } do |s|
s.item :appearance, safe_join([fa_icon('desktop fw'), t('settings.appearance')]), settings_preferences_appearance_path
s.item :notifications, safe_join([fa_icon('bell fw'), t('settings.notifications')]), settings_preferences_notifications_path
s.item :other, safe_join([fa_icon('cog fw'), t('preferences.other')]), settings_preferences_other_path
end
n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? }
n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? }
n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? && !self_destruct }
n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? && !self_destruct }
n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? && !self_destruct }
n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_path do |s|
s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_path, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities|^/disputes}
s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_methods_path, highlights_on: %r{/settings/two_factor_authentication|/settings/otp_authentication|/settings/security_keys}
s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_path
s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_path, if: -> { !self_destruct }
end
n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_path do |s|
s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_imports_path, if: -> { current_user.functional? }
s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_imports_path, if: -> { current_user.functional? && !self_destruct }
s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_path
end
n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: -> { current_user.can?(:invite_users) && current_user.functional? }
n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_path, if: -> { current_user.functional? }
n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: -> { current_user.can?(:invite_users) && current_user.functional? && !self_destruct }
n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_path, if: -> { current_user.functional? && !self_destruct }
n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_statuses_path, if: -> { current_user.can?(:manage_taxonomies) } do |s|
n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_statuses_path, if: -> { current_user.can?(:manage_taxonomies) && !self_destruct } do |s|
s.item :statuses, safe_join([fa_icon('comments-o fw'), t('admin.trends.statuses.title')]), admin_trends_statuses_path, highlights_on: %r{/admin/trends/statuses}
s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
end
n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks) } do |s|
n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks) && !self_destruct } do |s|
s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_path, highlights_on: %r{/admin/reports}, if: -> { current_user.can?(:manage_reports) }
s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_path(origin: 'local'), highlights_on: %r{/admin/accounts|/admin/pending_accounts|/admin/disputes|/admin/users}, if: -> { current_user.can?(:manage_users) }
s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path, if: -> { current_user.can?(:manage_invites) }
@ -49,7 +51,7 @@ SimpleNavigation::Configuration.run do |navigation|
s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_path, if: -> { current_user.can?(:view_audit_log) }
end
n.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), nil, if: -> { current_user.can?(:view_dashboard, :manage_settings, :manage_rules, :manage_announcements, :manage_custom_emojis, :manage_webhooks, :manage_federation) } do |s|
n.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), nil, if: -> { current_user.can?(:view_dashboard, :manage_settings, :manage_rules, :manage_announcements, :manage_custom_emojis, :manage_webhooks, :manage_federation) && !self_destruct } do |s|
s.item :dashboard, safe_join([fa_icon('tachometer fw'), t('admin.dashboard.title')]), admin_dashboard_path, if: -> { current_user.can?(:view_dashboard) }
s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), admin_settings_path, if: -> { current_user.can?(:manage_settings) }, highlights_on: %r{/admin/settings}
s.item :rules, safe_join([fa_icon('gavel fw'), t('admin.rules.title')]), admin_rules_path, highlights_on: %r{/admin/rules}, if: -> { current_user.can?(:manage_rules) }

1
config/sidekiq.yml
View File

@ -7,6 +7,7 @@
- [mailers, 2]
- [pull]
- [scheduler]
:scheduler:
:listened_queues_only: true
:schedule:

69
lib/mastodon/cli/main.rb
View File

@ -65,7 +65,6 @@ module Mastodon::CLI
desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
subcommand 'maintenance', Maintenance
option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC
Erase the server from the federation by broadcasting account delete
@ -92,55 +91,37 @@ module Mastodon::CLI
prompt = TTY::Prompt.new
if SelfDestructHelper.self_destruct?
prompt.ok('Self-destruct mode is already enabled for this Mastodon server')
pending_accounts = Account.local.without_suspended.count + Account.local.suspended.joins(:deletion_request).count
sidekiq_stats = Sidekiq::Stats.new
if pending_accounts.positive?
prompt.warn("#{pending_accounts} accounts are still pending deletion.")
elsif sidekiq_stats.enqueued.positive?
prompt.warn('Deletion notices are still being processed')
elsif sidekiq_stats.retry_size.positive?
prompt.warn('At least one delivery attempt for each deletion notice has been made, but some have failed and are scheduled for retry')
else
prompt.ok('Every deletion notice has been sent! You can safely delete all data and decomission your servers!')
end
exit(0)
end
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
unless dry_run?
prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
prompt.warn('This operation WILL NOT be reversible.')
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
prompt.warn('The deletion process itself may take a long time, and will be handled by Sidekiq, so do not shut it down until it has finished (you will be able to re-run this command to see the state of the self-destruct process).')
exit(1) if prompt.no?('Are you sure you want to proceed?')
end
inboxes = Account.inboxes
processed = 0
Setting.registrations_mode = 'none' unless dry_run?
if inboxes.empty?
Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless dry_run?
prompt.ok('It seems like your server has not federated with anything')
prompt.ok('You can shut it down and delete it any time')
return
end
prompt.warn('Do NOT interrupt this process...')
delete_account = lambda do |account|
payload = ActiveModelSerializers::SerializableResource.new(
account,
serializer: ActivityPub::DeleteActorSerializer,
adapter: ActivityPub::Adapter
).as_json
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
unless dry_run?
ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
[json, account.id, inbox_url]
end
account.suspend!(block_email: false)
end
processed += 1
end
Account.local.without_suspended.find_each { |account| delete_account.call(account) }
Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}")
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
self_destruct_value = Rails.application.message_verifier('self-destruct').generate(Rails.configuration.x.local_domain)
prompt.ok('To switch Mastodon to self-destruct mode, add the following variable to your evironment (e.g. by adding a line to your `.env.production`) and restart all Mastodon processes:')
prompt.ok(" SELF_DESTRUCT=#{self_destruct_value}")
prompt.ok("\nYou can re-run this command to see the state of the self-destruct process.")
rescue TTY::Reader::InputInterrupt
exit(1)
end