From b782f86b51e9d9653173b5fc79d5392e8ffebe31 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 30 Jan 2022 21:38:54 +0100 Subject: [PATCH 01/10] Fix some old migration scripts (#17394) * Fix some old migration scripts * Fix edge case in two-step migration from older releases --- ...3_remove_faux_remote_account_duplicates.rb | 40 +++++++++++++++++++ .../20190715164535_add_instance_actor.rb | 5 +++ .../20191007013357_update_pt_locales.rb | 4 ++ db/views/follow_recommendations_v01.sql | 4 +- db/views/follow_recommendations_v02.sql | 4 +- 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb b/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb index bd4f4c2a3..40537e9c9 100644 --- a/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb +++ b/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb @@ -1,6 +1,46 @@ class RemoveFauxRemoteAccountDuplicates < ActiveRecord::Migration[5.2] disable_ddl_transaction! + class StreamEntry < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :stream_entries + end + + class Status < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :statuses + has_many :favourites, inverse_of: :status, dependent: :destroy + has_many :mentions, dependent: :destroy, inverse_of: :status + end + + class Favourite < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :favourites + belongs_to :status, inverse_of: :favourites + end + + class Mention < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :mentions + belongs_to :status + end + + class Notification < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, optional: true + belongs_to :from_account, class_name: 'Account', optional: true + belongs_to :activity, polymorphic: true, optional: true + end + + class Account < ApplicationRecord + # Dummy class, to make migration possible across version changes + has_many :stream_entries, inverse_of: :account, dependent: :destroy + has_many :statuses, inverse_of: :account, dependent: :destroy + has_many :favourites, inverse_of: :account, dependent: :destroy + has_many :mentions, inverse_of: :account, dependent: :destroy + has_many :notifications, inverse_of: :account, dependent: :destroy + end + def up local_domain = Rails.configuration.x.local_domain diff --git a/db/migrate/20190715164535_add_instance_actor.rb b/db/migrate/20190715164535_add_instance_actor.rb index a26d54949..8c0301d69 100644 --- a/db/migrate/20190715164535_add_instance_actor.rb +++ b/db/migrate/20190715164535_add_instance_actor.rb @@ -1,4 +1,9 @@ class AddInstanceActor < ActiveRecord::Migration[5.2] + class Account < ApplicationRecord + # Dummy class, to make migration possible across version changes + validates :username, uniqueness: { scope: :domain, case_sensitive: false } + end + def up Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain) end diff --git a/db/migrate/20191007013357_update_pt_locales.rb b/db/migrate/20191007013357_update_pt_locales.rb index b7288d38a..9e8f8b424 100644 --- a/db/migrate/20191007013357_update_pt_locales.rb +++ b/db/migrate/20191007013357_update_pt_locales.rb @@ -1,4 +1,8 @@ class UpdatePtLocales < ActiveRecord::Migration[5.2] + class User < ApplicationRecord + # Dummy class, to make migration possible across version changes + end + disable_ddl_transaction! def up diff --git a/db/views/follow_recommendations_v01.sql b/db/views/follow_recommendations_v01.sql index 799abeaee..8295bbc0f 100644 --- a/db/views/follow_recommendations_v01.sql +++ b/db/views/follow_recommendations_v01.sql @@ -20,7 +20,7 @@ FROM ( HAVING count(follows.id) >= 5 UNION ALL SELECT accounts.id AS account_id, - sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank, + sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank, 'most_interactions' AS reason FROM status_stats INNER JOIN statuses ON statuses.id = status_stats.status_id @@ -32,7 +32,7 @@ FROM ( AND accounts.locked = 'f' AND accounts.discoverable = 't' GROUP BY accounts.id - HAVING sum(reblogs_count + favourites_count) >= 5 + HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5 ) t0 GROUP BY account_id ORDER BY rank DESC diff --git a/db/views/follow_recommendations_v02.sql b/db/views/follow_recommendations_v02.sql index 673c5cc85..f67c6eecf 100644 --- a/db/views/follow_recommendations_v02.sql +++ b/db/views/follow_recommendations_v02.sql @@ -18,7 +18,7 @@ FROM ( HAVING count(follows.id) >= 5 UNION ALL SELECT account_summaries.account_id AS account_id, - sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank, + sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank, 'most_interactions' AS reason FROM status_stats INNER JOIN statuses ON statuses.id = status_stats.status_id @@ -28,7 +28,7 @@ FROM ( AND account_summaries.sensitive = 'f' AND follow_recommendation_suppressions.id IS NULL GROUP BY account_summaries.account_id - HAVING sum(reblogs_count + favourites_count) >= 5 + HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5 ) t0 GROUP BY account_id ORDER BY rank DESC From 0dc103ea11c134897b74eac167404a2f83dec783 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 30 Jan 2022 22:34:54 +0100 Subject: [PATCH 02/10] Fix edge case in migration helpers that caused crash because of PostgreSQL quirks (#17398) --- lib/mastodon/migration_helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 39a6e0680..5bc903349 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -295,7 +295,7 @@ module Mastodon table = Arel::Table.new(table_name) total = estimate_rows_in_table(table_name).to_i - if total == 0 + if total < 1 count_arel = table.project(Arel.star.count.as('count')) count_arel = yield table, count_arel if block_given? From 959234c1e43f339abab093c62afaef96f6c52cb9 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Sun, 26 Dec 2021 06:52:24 +0900 Subject: [PATCH 03/10] Save bundle config as local (#17188) Some bundle options are saved as global user config and not project local. Specially, `deployment` must be saved as local config to be run on copied environment --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 962e5a8c9..cbcdb2034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,8 +54,8 @@ RUN npm install -g yarn && \ COPY Gemfile* package.json yarn.lock /opt/mastodon/ RUN cd /opt/mastodon && \ - bundle config set deployment 'true' && \ - bundle config set without 'development test' && \ + bundle config set --local deployment 'true' && \ + bundle config set --local without 'development test' && \ bundle install -j"$(nproc)" && \ yarn install --pure-lockfile From 2363b026e6e948fb53ee487804c3a084992ec13a Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 10 Sep 2021 15:43:28 +0200 Subject: [PATCH 04/10] Bump ruby-saml from 1.11.0 to 1.13.0 (#16723) Fixes #16720 --- Gemfile.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 201893d47..5b71445c8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -545,8 +545,9 @@ GEM rack (>= 1.1) rubocop (>= 1.7.0, < 2.0) ruby-progressbar (1.11.0) - ruby-saml (1.11.0) - nokogiri (>= 1.5.10) + ruby-saml (1.13.0) + nokogiri (>= 1.10.5) + rexml ruby2_keywords (0.0.4) rufus-scheduler (3.6.0) fugit (~> 1.1, >= 1.1.6) From 0ae91e45de7bbb26561e3557ad3276102ac6c489 Mon Sep 17 00:00:00 2001 From: Yusuke Nakamura Date: Fri, 12 Nov 2021 13:18:29 +0900 Subject: [PATCH 05/10] Build container image by GitHub Actions (#16973) * Build container image by GitHub Actions * Trigger docker build only pushed to main branch * Tweak tagging imgae - "edge" is the main branch - "latest" is the tagged latest release --- .github/workflows/build-image.yml | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/build-image.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 000000000..2ec6b3d4b --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,34 @@ +name: Build container image +on: + push: + branches: + - "main" + tags: + - "*" +jobs: + build-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: docker/setup-buildx-action@v1 + - uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/metadata-action@v3 + id: meta + with: + images: ghcr.io/${{ github.repository_owner }}/mastodon + flavor: | + latest=true + tags: | + type=edge,branch=main + type=semver,pattern={{ raw }} + - uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/mastodon:latest + cache-to: type=inline From c8301bcfc339d50d46c10440d8261add86a65635 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 14 Nov 2021 06:11:05 +0100 Subject: [PATCH 06/10] Change workflow to push to Docker Hub (#16980) --- .github/workflows/build-image.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 2ec6b3d4b..ae4c2aff9 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -13,15 +13,14 @@ jobs: - uses: docker/setup-buildx-action@v1 - uses: docker/login-action@v1 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - uses: docker/metadata-action@v3 id: meta with: - images: ghcr.io/${{ github.repository_owner }}/mastodon + images: tootsuite/mastodon flavor: | - latest=true + latest=auto tags: | type=edge,branch=main type=semver,pattern={{ raw }} @@ -30,5 +29,5 @@ jobs: context: . push: true tags: ${{ steps.meta.outputs.tags }} - cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/mastodon:latest + cache-from: type=registry,ref=tootsuite/mastodon:latest cache-to: type=inline From 2c83b9076d8d43714344bd428f796ec6805177b6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 16 Nov 2021 21:42:14 +0100 Subject: [PATCH 07/10] Add manual GitHub Actions runs (#17000) --- .github/workflows/build-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index ae4c2aff9..58f2813d3 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,5 +1,6 @@ name: Build container image on: + workflow_dispatch: push: branches: - "main" From 03f0e98b3277fd34d95f1501700deb44e85c9bf9 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 11 Aug 2021 17:48:42 +0200 Subject: [PATCH 08/10] Fix followers synchronization mechanism not working when URI has empty path (#16510) * Fix followers synchronization mechanism not working when URI has empty path To my knowledge, there is no current implementation on the fediverse that can use bare domains (e.g., actor is at https://example.org instead of something like https://example.org/actor) that also plans to support the followers synchronization mechanism. However, Mastodon's current implementation would exclude such accounts from followers list. Also adds tests and rename them to reflect the proper method names. * Move url prefix regexp to its own constant --- app/models/account.rb | 3 +- app/models/concerns/account_interactions.rb | 9 ++- app/workers/activitypub/delivery_worker.rb | 6 +- .../concerns/account_interactions_spec.rb | 63 +++++++++++-------- .../activitypub/delivery_worker_spec.rb | 2 +- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/app/models/account.rb b/app/models/account.rb index b2ce539df..2f2a55b55 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -60,6 +60,7 @@ class Account < ApplicationRecord USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i + URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ include AccountAssociations include AccountAvatar @@ -379,7 +380,7 @@ class Account < ApplicationRecord def synchronization_uri_prefix return 'local' if local? - @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//] + @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/" end class Field < ActiveModelSerializers::Model diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index 958f6c78e..763567f42 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -251,10 +251,13 @@ module AccountInteractions .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago) end - def remote_followers_hash(url_prefix) - Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do + def remote_followers_hash(url) + url_prefix = url[Account::URL_PREFIX_RE] + return if url_prefix.blank? + + Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do digest = "\x00" * 32 - followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri| + followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri| Xorcist.xor!(digest, Digest::SHA256.digest(uri)) end digest.unpack('H*')[0] diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 6c5a576a7..788f2cf80 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -44,11 +44,7 @@ class ActivityPub::DeliveryWorker end def synchronization_header - "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\"" - end - - def inbox_url_prefix - @inbox_url[/http(s?):\/\/[^\/]+\//] + "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(@inbox_url)}\", url=\"#{account_followers_synchronization_url(@source_account)}\"" end def perform_request diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb index 85fbf7e79..ca243ebc5 100644 --- a/spec/models/concerns/account_interactions_spec.rb +++ b/spec/models/concerns/account_interactions_spec.rb @@ -539,46 +539,57 @@ describe AccountInteractions do end end - describe '#followers_hash' do + describe '#remote_followers_hash' do let(:me) { Fabricate(:account, username: 'Me') } let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') } let(:remote_2) { Fabricate(:account, username: 'bob', domain: 'example.org', uri: 'https://example.org/users/bob') } - let(:remote_3) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') } + let(:remote_3) { Fabricate(:account, username: 'instance-actor', domain: 'example.org', uri: 'https://example.org') } + let(:remote_4) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') } before do remote_1.follow!(me) remote_2.follow!(me) remote_3.follow!(me) + remote_4.follow!(me) me.follow!(remote_1) end - context 'on a local user' do - it 'returns correct hash for remote domains' do - expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' - expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e' - end - - it 'invalidates cache as needed when removing or adding followers' do - expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' - remote_1.unfollow!(me) - expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff' - remote_1.follow!(me) - expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' - end + it 'returns correct hash for remote domains' do + expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7' + expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e' + expect(me.remote_followers_hash('https://foo.org.evil.com/')).to eq '0000000000000000000000000000000000000000000000000000000000000000' + expect(me.remote_followers_hash('https://foo')).to eq '0000000000000000000000000000000000000000000000000000000000000000' end - context 'on a remote user' do - it 'returns correct hash for remote domains' do - expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) - end + it 'invalidates cache as needed when removing or adding followers' do + expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7' + remote_3.unfollow!(me) + expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' + remote_1.unfollow!(me) + expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff' + remote_1.follow!(me) + expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' + end + end - it 'invalidates cache as needed when removing or adding followers' do - expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) - me.unfollow!(remote_1) - expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000' - me.follow!(remote_1) - expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) - end + describe '#local_followers_hash' do + let(:me) { Fabricate(:account, username: 'Me') } + let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') } + + before do + me.follow!(remote_1) + end + + it 'returns correct hash for local users' do + expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) + end + + it 'invalidates cache as needed when removing or adding followers' do + expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) + me.unfollow!(remote_1) + expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000' + me.follow!(remote_1) + expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) end end diff --git a/spec/workers/activitypub/delivery_worker_spec.rb b/spec/workers/activitypub/delivery_worker_spec.rb index f4633731e..d39393d50 100644 --- a/spec/workers/activitypub/delivery_worker_spec.rb +++ b/spec/workers/activitypub/delivery_worker_spec.rb @@ -11,7 +11,7 @@ describe ActivityPub::DeliveryWorker do let(:payload) { 'test' } before do - allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/').and_return('somehash') + allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/api').and_return('somehash') end describe 'perform' do From d722222fe15fc7b285fe5f90f4827ce68cee6291 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 30 Jan 2022 23:50:08 +0100 Subject: [PATCH 09/10] Add more advanced migration tests (#17393) - populate the database with some data when testing migrations - try both one-step and two-step migrations (`SKIP_POST_DEPLOYMENT_MIGRATIONS`) --- .circleci/config.yml | 42 +++++++++- lib/tasks/tests.rake | 181 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 lib/tasks/tests.rake diff --git a/.circleci/config.yml b/.circleci/config.yml index 42c058295..e1c43ff80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -167,8 +167,45 @@ jobs: name: Create database command: ./bin/rails db:create - run: - name: Run migrations + command: ./bin/rails db:migrate VERSION=20171010025614 + name: Run migrations up to v2.0.0 + - run: + command: ./bin/rails tests:migrations:populate_v2 + name: Populate database with test data + - run: command: ./bin/rails db:migrate + name: Run all remaining migrations + + test-two-step-migrations: + <<: *defaults + docker: + - image: circleci/ruby:2.7-buster-node + environment: *ruby_environment + - image: circleci/postgres:12.2 + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + - image: circleci/redis:5-alpine + steps: + - *attach_workspace + - *install_system_dependencies + - run: + command: ./bin/rails db:create + name: Create database + - run: + command: ./bin/rails db:migrate VERSION=20171010025614 + name: Run migrations up to v2.0.0 + - run: + command: ./bin/rails tests:migrations:populate_v2 + name: Populate database with test data + - run: + command: ./bin/rails db:migrate + name: Run all pre-deployment migrations + evironment: + SKIP_POST_DEPLOYMENT_MIGRATIONS: true + - run: + command: ./bin/rails db:migrate + name: Run all post-deployment remaining migrations test-ruby2.7: <<: *defaults @@ -238,6 +275,9 @@ workflows: - test-migrations: requires: - install-ruby2.7 + - test-two-step-migrations: + requires: + - install-ruby2.7 - test-ruby2.7: requires: - install-ruby2.7 diff --git a/lib/tasks/tests.rake b/lib/tasks/tests.rake new file mode 100644 index 000000000..0f38b50e3 --- /dev/null +++ b/lib/tasks/tests.rake @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +namespace :tests do + namespace :migrations do + desc 'Populate the database with test data for 2.0.0' + task populate_v2: :environment do + admin_key = OpenSSL::PKey::RSA.new(2048) + user_key = OpenSSL::PKey::RSA.new(2048) + remote_key = OpenSSL::PKey::RSA.new(2048) + remote_key2 = OpenSSL::PKey::RSA.new(2048) + remote_key3 = OpenSSL::PKey::RSA.new(2048) + admin_private_key = ActiveRecord::Base.connection.quote(admin_key.to_pem) + admin_public_key = ActiveRecord::Base.connection.quote(admin_key.public_key.to_pem) + user_private_key = ActiveRecord::Base.connection.quote(user_key.to_pem) + user_public_key = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem) + remote_public_key = ActiveRecord::Base.connection.quote(remote_key.public_key.to_pem) + remote_public_key2 = ActiveRecord::Base.connection.quote(remote_key2.public_key.to_pem) + remote_public_key_ap = ActiveRecord::Base.connection.quote(remote_key3.public_key.to_pem) + local_domain = ActiveRecord::Base.connection.quote(Rails.configuration.x.local_domain) + + ActiveRecord::Base.connection.execute(<<~SQL) + -- accounts + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at) + VALUES + (1, 'admin', NULL, #{admin_private_key}, #{admin_public_key}, now(), now()), + (2, 'user', NULL, #{user_private_key}, #{user_public_key}, now(), now()); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at, remote_url, salmon_url) + VALUES + (3, 'remote', 'remote.com', NULL, #{remote_public_key}, now(), now(), + 'https://remote.com/@remote', 'https://remote.com/salmon/1'), + (4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(), + 'https://remote.com/@Remote', 'https://remote.com/salmon/1'), + (5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now(), now(), + 'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1'); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url) + VALUES + (6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(), + 1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers'); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at) + VALUES + (7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()), + (8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now()); + + -- users + + INSERT INTO "users" + (id, account_id, email, created_at, updated_at, admin) + VALUES + (1, 1, 'admin@localhost', now(), now(), true), + (2, 2, 'user@localhost', now(), now(), false); + + INSERT INTO "users" + (id, account_id, email, created_at, updated_at, admin, locale) + VALUES + (3, 7, 'ptuser@localhost', now(), now(), false, 'pt'); + + -- statuses + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (1, 1, 'test', now(), now()), + (2, 1, '@remote@remote.com hello', now(), now()), + (3, 1, '@Remote@remote.com hello', now(), now()), + (4, 1, '@REMOTE@remote.com hello', now(), now()); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at, uri, local) + VALUES + (5, 1, 'activitypub status', now(), now(), 'https://localhost/users/admin/statuses/4', true); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (6, 3, 'test', now(), now()); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at, in_reply_to_id, in_reply_to_account_id) + VALUES + (7, 4, '@admin hello', now(), now(), 3, 1); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (8, 5, 'test', now(), now()); + + INSERT INTO "statuses" + (id, account_id, reblog_of_id, created_at, updated_at) + VALUES + (9, 1, 2, now(), now()); + + -- mentions (from previous statuses) + + INSERT INTO "mentions" + (status_id, account_id, created_at, updated_at) + VALUES + (2, 3, now(), now()), + (3, 4, now(), now()), + (4, 5, now(), now()); + + -- stream entries + + INSERT INTO "stream_entries" + (activity_id, account_id, activity_type, created_at, updated_at) + VALUES + (1, 1, 'status', now(), now()), + (2, 1, 'status', now(), now()), + (3, 1, 'status', now(), now()), + (4, 1, 'status', now(), now()), + (5, 1, 'status', now(), now()), + (6, 3, 'status', now(), now()), + (7, 4, 'status', now(), now()), + (8, 5, 'status', now(), now()), + (9, 1, 'status', now(), now()); + + + -- custom emoji + + INSERT INTO "custom_emojis" + (shortcode, created_at, updated_at) + VALUES + ('test', now(), now()), + ('Test', now(), now()), + ('blobcat', now(), now()); + + INSERT INTO "custom_emojis" + (shortcode, domain, uri, created_at, updated_at) + VALUES + ('blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()), + ('blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()), + ('Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now()); + + -- favourites + + INSERT INTO "favourites" + (account_id, status_id, created_at, updated_at) + VALUES + (1, 1, now(), now()), + (1, 7, now(), now()), + (4, 1, now(), now()), + (3, 1, now(), now()), + (5, 1, now(), now()); + + -- pinned statuses + + INSERT INTO "status_pins" + (account_id, status_id, created_at, updated_at) + VALUES + (1, 1, now(), now()), + (3, 6, now(), now()), + (4, 7, now(), now()); + + -- follows + + INSERT INTO "follows" + (account_id, target_account_id, created_at, updated_at) + VALUES + (1, 5, now(), now()), + (6, 2, now(), now()), + (5, 2, now(), now()), + (6, 1, now(), now()); + + -- follow requests + + INSERT INTO "follow_requests" + (account_id, target_account_id, created_at, updated_at) + VALUES + (2, 5, now(), now()), + (5, 1, now(), now()); + SQL + end + end +end From 1c8c318281ba67ab60c70fab27204296b0af74b1 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 31 Jan 2022 10:36:48 +0100 Subject: [PATCH 10/10] Bump version to 3.4.5 --- CHANGELOG.md | 12 ++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deca765fa..9deff5a0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ Changelog All notable changes to this project will be documented in this file. +## [3.4.5] - 2022-01-31 +### Added +- Add more advanced migration tests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17393)) +- Add github workflow to build Docker images ([unasuke](https://github.com/mastodon/mastodon/pull/16973), [Gargron](https://github.com/mastodon/mastodon/pull/16980), [Gargron](https://github.com/mastodon/mastodon/pull/17000)) + +### Fixed +- Fix some old migrations failing when skipping releases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17394)) +- Fix migrations script failing in certain edge cases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17398)) +- Fix Docker build ([tribela](https://github.com/mastodon/mastodon/pull/17188)) +- Fix Ruby 3.0 dependencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16723)) +- Fix followers synchronization mechanism ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16510)) + ## [3.4.4] - 2021-11-26 ### Fixed diff --git a/docker-compose.yml b/docker-compose.yml index 52eea7a74..288a4fdf6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: web: build: . - image: tootsuite/mastodon + image: tootsuite/mastodon:v3.4.5 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -63,7 +63,7 @@ services: streaming: build: . - image: tootsuite/mastodon + image: tootsuite/mastodon:v3.4.5 restart: always env_file: .env.production command: node ./streaming @@ -80,7 +80,7 @@ services: sidekiq: build: . - image: tootsuite/mastodon + image: tootsuite/mastodon:v3.4.5 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 493479977..1ba45aeca 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 4 + 5 end def flags