More duplicates in cli maintenance spec, misc bug fixes (#28449)

lib/mastodon/cli/maintenance.rb

@@ -40,6 +40,10 @@ module Mastodon::CLI
     class BulkImport < ApplicationRecord; end
     class SoftwareUpdate < ApplicationRecord; end
 
+    class DomainBlock < ApplicationRecord
+      scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) }
+    end
+
     class PreviewCard < ApplicationRecord
       self.inheritance_column = false
     end
@@ -249,19 +253,7 @@ module Mastodon::CLI
 
       say 'Deduplicating user records…'
 
-      # Deduplicating email
-      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
-        users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
-        ref_user = users.shift
-        say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
-        say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
-        say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
-
-        users.each_with_index do |user, index|
-          user.update!(email: "#{index} " + user.email)
-        end
-      end
-
+      deduplicate_users_process_email
       deduplicate_users_process_confirmation_token
       deduplicate_users_process_remember_token
       deduplicate_users_process_password_token
@@ -280,6 +272,20 @@ module Mastodon::CLI
       ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753
     end
 
+    def deduplicate_users_process_email
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
+        users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
+        ref_user = users.shift
+        say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
+        say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
+        say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
+
+        users.each_with_index do |user, index|
+          user.update!(email: "#{index} " + user.email)
+        end
+      end
+    end
+
     def deduplicate_users_process_confirmation_token
       ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
         users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
@@ -571,7 +577,7 @@ module Mastodon::CLI
 
       say 'Deduplicating webhooks…'
       ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
-        Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+        Webhook.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
       end
 
       say 'Restoring webhooks indexes…'
@@ -604,11 +610,7 @@ module Mastodon::CLI
 
       say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
 
-      ref_id = ask('Account to keep unchanged:') do |q|
-        q.required true
-        q.default 0
-        q.convert :int
-      end
+      ref_id = ask('Account to keep unchanged:', required: true, default: 0).to_i
 
       accounts.delete_at(ref_id)
 

spec/lib/mastodon/cli/maintenance_spec.rb

@@ -62,6 +62,7 @@ describe Mastodon::CLI::Maintenance do
       context 'with duplicate accounts' do
         before do
           prepare_duplicate_data
+          choose_local_account_to_keep
         end
 
         let(:duplicate_account_username) { 'username' }
@@ -71,21 +72,37 @@ describe Mastodon::CLI::Maintenance do
           expect { subject }
             .to output_results(
               'Deduplicating accounts',
+              'Multiple local accounts were found for',
               'Restoring index_accounts_on_username_and_domain_lower',
               'Reindexing textual indexes on accounts…',
               'Finished!'
             )
-            .and change(duplicate_accounts, :count).from(2).to(1)
+            .and change(duplicate_remote_accounts, :count).from(2).to(1)
+            .and change(duplicate_local_accounts, :count).from(2).to(1)
         end
 
-        def duplicate_accounts
+        def duplicate_remote_accounts
           Account.where(username: duplicate_account_username, domain: duplicate_account_domain)
         end
 
+        def duplicate_local_accounts
+          Account.where(username: duplicate_account_username, domain: nil)
+        end
+
         def prepare_duplicate_data
           ActiveRecord::Base.connection.remove_index :accounts, name: :index_accounts_on_username_and_domain_lower
-          Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain)
-          Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false)
+          _remote_account = Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain)
+          _remote_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false)
+          _local_account = Fabricate(:account, username: duplicate_account_username, domain: nil)
+          _local_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: nil).save(validate: false)
+        end
+
+        def choose_local_account_to_keep
+          allow(cli.shell)
+            .to receive(:ask)
+            .with(/Account to keep unchanged/, anything)
+            .and_return('0')
+            .once
         end
       end
 
@@ -175,6 +192,407 @@ describe Mastodon::CLI::Maintenance do
         end
       end
 
+      context 'with duplicate account_domain_blocks' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:duplicate_domain) { 'example.host' }
+        let(:account) { Fabricate(:account) }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Removing duplicate account domain blocks',
+              'Restoring account domain blocks indexes',
+              'Finished!'
+            )
+            .and change(duplicate_account_domain_blocks, :count).from(2).to(1)
+        end
+
+        def duplicate_account_domain_blocks
+          AccountDomainBlock.where(account: account, domain: duplicate_domain)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :account_domain_blocks, [:account_id, :domain]
+          Fabricate(:account_domain_block, account: account, domain: duplicate_domain)
+          Fabricate.build(:account_domain_block, account: account, domain: duplicate_domain).save(validate: false)
+        end
+      end
+
+      context 'with duplicate announcement_reactions' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:account) { Fabricate(:account) }
+        let(:announcement) { Fabricate(:announcement) }
+        let(:name) { Fabricate(:custom_emoji).shortcode }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Removing duplicate announcement reactions',
+              'Restoring announcement_reactions indexes',
+              'Finished!'
+            )
+            .and change(duplicate_announcement_reactions, :count).from(2).to(1)
+        end
+
+        def duplicate_announcement_reactions
+          AnnouncementReaction.where(account: account, announcement: announcement, name: name)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :announcement_reactions, [:account_id, :announcement_id, :name]
+          Fabricate(:announcement_reaction, account: account, announcement: announcement, name: name)
+          Fabricate.build(:announcement_reaction, account: account, announcement: announcement, name: name).save(validate: false)
+        end
+      end
+
+      context 'with duplicate conversations' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:uri) { 'https://example.host/path' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating conversations',
+              'Restoring conversations indexes',
+              'Finished!'
+            )
+            .and change(duplicate_conversations, :count).from(2).to(1)
+        end
+
+        def duplicate_conversations
+          Conversation.where(uri: uri)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :conversations, :uri
+          Fabricate(:conversation, uri: uri)
+          Fabricate.build(:conversation, uri: uri).save(validate: false)
+        end
+      end
+
+      context 'with duplicate custom_emojis' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:duplicate_shortcode) { 'wowzers' }
+        let(:duplicate_domain) { 'example.host' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating custom_emojis',
+              'Restoring custom_emojis indexes',
+              'Finished!'
+            )
+            .and change(duplicate_custom_emojis, :count).from(2).to(1)
+        end
+
+        def duplicate_custom_emojis
+          CustomEmoji.where(shortcode: duplicate_shortcode, domain: duplicate_domain)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :custom_emojis, [:shortcode, :domain]
+          Fabricate(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain)
+          Fabricate.build(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain).save(validate: false)
+        end
+      end
+
+      context 'with duplicate custom_emoji_categories' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:duplicate_name) { 'name_value' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating custom_emoji_categories',
+              'Restoring custom_emoji_categories indexes',
+              'Finished!'
+            )
+            .and change(duplicate_custom_emoji_categories, :count).from(2).to(1)
+        end
+
+        def duplicate_custom_emoji_categories
+          CustomEmojiCategory.where(name: duplicate_name)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :custom_emoji_categories, :name
+          Fabricate(:custom_emoji_category, name: duplicate_name)
+          Fabricate.build(:custom_emoji_category, name: duplicate_name).save(validate: false)
+        end
+      end
+
+      context 'with duplicate domain_allows' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:domain) { 'example.host' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating domain_allows',
+              'Restoring domain_allows indexes',
+              'Finished!'
+            )
+            .and change(duplicate_domain_allows, :count).from(2).to(1)
+        end
+
+        def duplicate_domain_allows
+          DomainAllow.where(domain: domain)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :domain_allows, :domain
+          Fabricate(:domain_allow, domain: domain)
+          Fabricate.build(:domain_allow, domain: domain).save(validate: false)
+        end
+      end
+
+      context 'with duplicate domain_blocks' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:domain) { 'example.host' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating domain_blocks',
+              'Restoring domain_blocks indexes',
+              'Finished!'
+            )
+            .and change(duplicate_domain_blocks, :count).from(2).to(1)
+        end
+
+        def duplicate_domain_blocks
+          DomainBlock.where(domain: domain)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :domain_blocks, :domain
+          Fabricate(:domain_block, domain: domain)
+          Fabricate.build(:domain_block, domain: domain).save(validate: false)
+        end
+      end
+
+      context 'with duplicate email_domain_blocks' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:domain) { 'example.host' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating email_domain_blocks',
+              'Restoring email_domain_blocks indexes',
+              'Finished!'
+            )
+            .and change(duplicate_email_domain_blocks, :count).from(2).to(1)
+        end
+
+        def duplicate_email_domain_blocks
+          EmailDomainBlock.where(domain: domain)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :email_domain_blocks, :domain
+          Fabricate(:email_domain_block, domain: domain)
+          Fabricate.build(:email_domain_block, domain: domain).save(validate: false)
+        end
+      end
+
+      context 'with duplicate media_attachments' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:shortcode) { 'codenam' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating media_attachments',
+              'Restoring media_attachments indexes',
+              'Finished!'
+            )
+            .and change(duplicate_media_attachments, :count).from(2).to(1)
+        end
+
+        def duplicate_media_attachments
+          MediaAttachment.where(shortcode: shortcode)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :media_attachments, :shortcode
+          Fabricate(:media_attachment, shortcode: shortcode)
+          Fabricate.build(:media_attachment, shortcode: shortcode).save(validate: false)
+        end
+      end
+
+      context 'with duplicate preview_cards' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:url) { 'https://example.host/path' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating preview_cards',
+              'Restoring preview_cards indexes',
+              'Finished!'
+            )
+            .and change(duplicate_preview_cards, :count).from(2).to(1)
+        end
+
+        def duplicate_preview_cards
+          PreviewCard.where(url: url)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :preview_cards, :url
+          Fabricate(:preview_card, url: url)
+          Fabricate.build(:preview_card, url: url).save(validate: false)
+        end
+      end
+
+      context 'with duplicate statuses' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:uri) { 'https://example.host/path' }
+        let(:account) { Fabricate(:account) }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating statuses',
+              'Restoring statuses indexes',
+              'Finished!'
+            )
+            .and change(duplicate_statuses, :count).from(2).to(1)
+        end
+
+        def duplicate_statuses
+          Status.where(uri: uri)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :statuses, :uri
+          Fabricate(:status, account: account, uri: uri)
+          duplicate = Fabricate.build(:status, account: account, uri: uri)
+          duplicate.save(validate: false)
+          Fabricate(:status_pin, account: account, status: duplicate)
+          Fabricate(:status, in_reply_to_id: duplicate.id)
+          Fabricate(:status, reblog_of_id: duplicate.id)
+        end
+      end
+
+      context 'with duplicate tags' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:name) { 'tagname' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating tags',
+              'Restoring tags indexes',
+              'Finished!'
+            )
+            .and change(duplicate_tags, :count).from(2).to(1)
+        end
+
+        def duplicate_tags
+          Tag.where(name: name)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :tags, name: 'index_tags_on_name_lower_btree'
+          Fabricate(:tag, name: name)
+          Fabricate.build(:tag, name: name).save(validate: false)
+        end
+      end
+
+      context 'with duplicate webauthn_credentials' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:external_id) { '123_123_123' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating webauthn_credentials',
+              'Restoring webauthn_credentials indexes',
+              'Finished!'
+            )
+            .and change(duplicate_webauthn_credentials, :count).from(2).to(1)
+        end
+
+        def duplicate_webauthn_credentials
+          WebauthnCredential.where(external_id: external_id)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :webauthn_credentials, :external_id
+          Fabricate(:webauthn_credential, external_id: external_id)
+          Fabricate.build(:webauthn_credential, external_id: external_id).save(validate: false)
+        end
+      end
+
+      context 'with duplicate webhooks' do
+        before do
+          prepare_duplicate_data
+        end
+
+        let(:url) { 'https://example.host/path' }
+
+        it 'runs the deduplication process' do
+          expect { subject }
+            .to output_results(
+              'Deduplicating webhooks',
+              'Restoring webhooks indexes',
+              'Finished!'
+            )
+            .and change(duplicate_webhooks, :count).from(2).to(1)
+        end
+
+        def duplicate_webhooks
+          Webhook.where(url: url)
+        end
+
+        def prepare_duplicate_data
+          ActiveRecord::Base.connection.remove_index :webhooks, :url
+          Fabricate(:webhook, url: url)
+          Fabricate.build(:webhook, url: url).save(validate: false)
+        end
+      end
+
       def agree_to_backup_warning
         allow(cli.shell)
           .to receive(:yes?)