From 54ae3d5ca526c9aaaed6679ae89bae28a0e84b4f Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 22 Feb 2024 14:38:11 +0100 Subject: [PATCH 01/46] Add basic CI to 4.1 branch (#29351) --- .github/workflows/test-ruby.yml | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .github/workflows/test-ruby.yml diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml new file mode 100644 index 000000000..90467e46d --- /dev/null +++ b/.github/workflows/test-ruby.yml @@ -0,0 +1,151 @@ +name: Ruby Testing + +on: + push: + branches-ignore: + - 'dependabot/**' + - 'renovate/**' + pull_request: + +env: + BUNDLE_CLEAN: true + BUNDLE_FROZEN: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + mode: + - production + - test + env: + RAILS_ENV: ${{ matrix.mode }} + BUNDLE_WITH: ${{ matrix.mode }} + OTP_SECRET: precompile_placeholder + SECRET_KEY_BASE: precompile_placeholder + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + cache: yarn + node-version-file: '.nvmrc' + + - name: Install native Ruby dependencies + run: | + sudo apt-get update + sudo apt-get install -y libicu-dev libidn11-dev + + - name: Set up bundler cache + uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + bundler-cache: true + + - run: yarn --frozen-lockfile --production + - name: Precompile assets + # Previously had set this, but it's not supported + # export NODE_OPTIONS=--openssl-legacy-provider + run: |- + ./bin/rails assets:precompile + + - uses: actions/upload-artifact@v3 + if: matrix.mode == 'test' + with: + path: |- + ./public/assets + ./public/packs-test + name: ${{ github.sha }} + retention-days: 0 + + test: + runs-on: ubuntu-latest + + needs: + - build + + services: + postgres: + image: postgres:14-alpine + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + env: + DB_HOST: localhost + DB_USER: postgres + DB_PASS: postgres + DISABLE_SIMPLECOV: true + RAILS_ENV: test + ALLOW_NOPAM: true + PAM_ENABLED: true + PAM_DEFAULT_SERVICE: pam_test + PAM_CONTROLLED_SERVICE: pam_test_controlled + OIDC_ENABLED: true + OIDC_SCOPE: read + SAML_ENABLED: true + CAS_ENABLED: true + BUNDLE_WITH: 'pam_authentication test' + CI_JOBS: ${{ matrix.ci_job }}/4 + + strategy: + fail-fast: false + matrix: + ruby-version: + - '.ruby-version' + ci_job: + - 1 + - 2 + - 3 + - 4 + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v3 + with: + path: './public' + name: ${{ github.sha }} + + - name: Update package index + run: sudo apt-get update + + - name: Install native Ruby dependencies + run: sudo apt-get install -y libicu-dev libidn11-dev + + - name: Install additional system dependencies + run: sudo apt-get install -y ffmpeg imagemagick libpam-dev + + - name: Set up bundler cache + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version}} + bundler-cache: true + + - name: Load database schema + run: './bin/rails db:create db:schema:load db:seed' From 1629ac4c811f215a565647a43144bd88be7cc7a2 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 22 Feb 2024 14:52:07 +0100 Subject: [PATCH 02/46] Update dependencies (#29350) --- Gemfile.lock | 118 +++++++++++++++++++++++++-------------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index e08a98062..29a437fb0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,40 +10,40 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + actioncable (6.1.7.7) + actionpack (= 6.1.7.7) + activesupport (= 6.1.7.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionmailbox (6.1.7.7) + actionpack (= 6.1.7.7) + activejob (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) mail (>= 2.7.1) - actionmailer (6.1.7.6) - actionpack (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionmailer (6.1.7.7) + actionpack (= 6.1.7.7) + actionview (= 6.1.7.7) + activejob (= 6.1.7.7) + activesupport (= 6.1.7.7) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.7.6) - actionview (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionpack (6.1.7.7) + actionview (= 6.1.7.7) + activesupport (= 6.1.7.7) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.7.6) - actionpack (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actiontext (6.1.7.7) + actionpack (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) nokogiri (>= 1.8.5) - actionview (6.1.7.6) - activesupport (= 6.1.7.6) + actionview (6.1.7.7) + activesupport (= 6.1.7.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -54,22 +54,22 @@ GEM case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_query_trace (1.8) - activejob (6.1.7.6) - activesupport (= 6.1.7.6) + activejob (6.1.7.7) + activesupport (= 6.1.7.7) globalid (>= 0.3.6) - activemodel (6.1.7.6) - activesupport (= 6.1.7.6) - activerecord (6.1.7.6) - activemodel (= 6.1.7.6) - activesupport (= 6.1.7.6) - activestorage (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activesupport (= 6.1.7.6) + activemodel (6.1.7.7) + activesupport (= 6.1.7.7) + activerecord (6.1.7.7) + activemodel (= 6.1.7.7) + activesupport (= 6.1.7.7) + activestorage (6.1.7.7) + actionpack (= 6.1.7.7) + activejob (= 6.1.7.7) + activerecord (= 6.1.7.7) + activesupport (= 6.1.7.7) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.7.6) + activesupport (6.1.7.7) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -173,7 +173,7 @@ GEM cocoon (1.2.15) coderay (1.1.3) color_diff (0.1) - concurrent-ruby (1.2.2) + concurrent-ruby (1.2.3) connection_pool (2.3.0) cose (1.2.1) cbor (~> 0.5.9) @@ -183,7 +183,7 @@ GEM crass (1.0.6) css_parser (1.12.0) addressable - date (3.3.3) + date (3.3.4) debug_inspector (1.0.0) devise (4.8.1) bcrypt (~> 3.0) @@ -416,11 +416,11 @@ GEM net-ldap (0.17.1) net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout net-scp (4.0.0.rc1) net-ssh (>= 2.6.5, < 8.0.0) - net-smtp (0.3.3) + net-smtp (0.3.4) net-protocol net-ssh (7.0.1) nio4r (2.5.9) @@ -497,7 +497,7 @@ GEM activesupport (>= 3.0.0) raabro (1.4.0) racc (1.7.3) - rack (2.2.8) + rack (2.2.8.1) rack-attack (6.6.1) rack (>= 1.0, < 3) rack-cors (1.1.1) @@ -512,20 +512,20 @@ GEM rack rack-test (2.0.2) rack (>= 1.3) - rails (6.1.7.6) - actioncable (= 6.1.7.6) - actionmailbox (= 6.1.7.6) - actionmailer (= 6.1.7.6) - actionpack (= 6.1.7.6) - actiontext (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activemodel (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + rails (6.1.7.7) + actioncable (= 6.1.7.7) + actionmailbox (= 6.1.7.7) + actionmailer (= 6.1.7.7) + actionpack (= 6.1.7.7) + actiontext (= 6.1.7.7) + actionview (= 6.1.7.7) + activejob (= 6.1.7.7) + activemodel (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) bundler (>= 1.15.0) - railties (= 6.1.7.6) + railties (= 6.1.7.7) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -541,9 +541,9 @@ GEM railties (>= 6.0.0, < 7) rails-settings-cached (0.6.6) rails (>= 4.2.0) - railties (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + railties (6.1.7.7) + actionpack (= 6.1.7.7) + activesupport (= 6.1.7.7) method_source rake (>= 12.2) thor (~> 1.0) @@ -753,7 +753,7 @@ GEM xorcist (1.1.3) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.12) + zeitwerk (2.6.13) PLATFORMS ruby From 2fc87611be94cd5bfba5cc08d716fff051a18838 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 22 Feb 2024 14:39:42 +0100 Subject: [PATCH 03/46] Automatically switch from open to approved registrations in absence of moderators (#29337) --- app/mailers/admin_mailer.rb | 9 +++ .../auto_close_registrations.text.erb | 3 + .../auto_close_registrations_scheduler.rb | 33 ++++++++++ config/locales/en.yml | 3 + config/sidekiq.yml | 4 ++ ...auto_close_registrations_scheduler_spec.rb | 60 +++++++++++++++++++ 6 files changed, 112 insertions(+) create mode 100644 app/views/admin_mailer/auto_close_registrations.text.erb create mode 100644 app/workers/scheduler/auto_close_registrations_scheduler.rb create mode 100644 spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index bc6d87ae6..c889c91d6 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -47,4 +47,13 @@ class AdminMailer < ApplicationMailer mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance) end end + + def auto_close_registrations(recipient) + @me = recipient + @instance = Rails.configuration.x.local_domain + + locale_for_account(@me) do + mail to: @me.user_email, subject: I18n.t('admin_mailer.auto_close_registrations.subject', instance: @instance) + end + end end diff --git a/app/views/admin_mailer/auto_close_registrations.text.erb b/app/views/admin_mailer/auto_close_registrations.text.erb new file mode 100644 index 000000000..c0f848692 --- /dev/null +++ b/app/views/admin_mailer/auto_close_registrations.text.erb @@ -0,0 +1,3 @@ +<%= raw t('admin_mailer.auto_close_registrations.body', instance: @instance) %> + +<%= raw t('application_mailer.view')%> <%= admin_settings_registrations_url %> diff --git a/app/workers/scheduler/auto_close_registrations_scheduler.rb b/app/workers/scheduler/auto_close_registrations_scheduler.rb new file mode 100644 index 000000000..979b336fa --- /dev/null +++ b/app/workers/scheduler/auto_close_registrations_scheduler.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Scheduler::AutoCloseRegistrationsScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + # Automatically switch away from open registrations if no + # moderator had any activity in that period of time + OPEN_REGISTRATIONS_MODERATOR_THRESHOLD = 1.week + UserTrackingConcern::SIGN_IN_UPDATE_FREQUENCY + + def perform + return if Rails.configuration.x.email_domains_whitelist.present? || ENV['DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS'] == 'true' + return unless Setting.registrations_mode == 'open' + + switch_to_approval_mode! unless active_moderators? + end + + private + + def active_moderators? + User.those_who_can(:manage_reports).exists?(current_sign_in_at: OPEN_REGISTRATIONS_MODERATOR_THRESHOLD.ago...) + end + + def switch_to_approval_mode! + Setting.registrations_mode = 'approved' + + User.those_who_can(:view_devops).includes(:account).find_each do |user| + AdminMailer.auto_close_registrations(recipient: user.account).deliver_later + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 8aab40333..cb1ca670c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -905,6 +905,9 @@ en: title: Webhooks webhook: Webhook admin_mailer: + auto_close_registrations: + body: Due to a lack of recent moderator activity, registrations on %{instance} have been automatically switched to requiring manual review, to prevent %{instance} from being used as a platform for potential bad actors. You can switch it back to open registrations at any time. + subject: Registrations for %{instance} have been automatically switched to requiring approval new_appeal: actions: delete_statuses: to delete their posts diff --git a/config/sidekiq.yml b/config/sidekiq.yml index b8739aab3..dbca16791 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -58,3 +58,7 @@ interval: 1 minute class: Scheduler::SuspendedUserCleanupScheduler queue: scheduler + auto_close_registrations_scheduler: + interval: 1 hour + class: Scheduler::AutoCloseRegistrationsScheduler + queue: scheduler diff --git a/spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb b/spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb new file mode 100644 index 000000000..c0c50b128 --- /dev/null +++ b/spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Scheduler::AutoCloseRegistrationsScheduler do + subject { described_class.new } + + describe '#perform' do + let(:moderator_activity_date) { Time.now.utc } + + before do + Fabricate(:user, role: UserRole.find_by(name: 'Owner'), current_sign_in_at: 10.years.ago) + Fabricate(:user, role: UserRole.find_by(name: 'Moderator'), current_sign_in_at: moderator_activity_date) + end + + context 'when registrations are open' do + before do + Setting.registrations_mode = 'open' + end + + context 'when a moderator has logged in recently' do + let(:moderator_activity_date) { Time.now.utc } + + it 'does not change registrations mode' do + expect { subject.perform }.to_not change(Setting, :registrations_mode) + end + end + + context 'when a moderator has not recently signed in' do + let(:moderator_activity_date) { 1.year.ago } + + it 'changes registrations mode from open to approved' do + expect { subject.perform }.to change(Setting, :registrations_mode).from('open').to('approved') + end + end + end + + context 'when registrations are closed' do + before do + Setting.registrations_mode = 'none' + end + + context 'when a moderator has logged in recently' do + let(:moderator_activity_date) { Time.now.utc } + + it 'does not change registrations mode' do + expect { subject.perform }.to_not change(Setting, :registrations_mode) + end + end + + context 'when a moderator has not recently signed in' do + let(:moderator_activity_date) { 1.year.ago } + + it 'does not change registrations mode' do + expect { subject.perform }.to_not change(Setting, :registrations_mode) + end + end + end + end +end From c7312411b84498ded85b64d0600823cd4e56c850 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 22 Feb 2024 18:15:38 +0100 Subject: [PATCH 04/46] Fix auto-close email being sent to users with devops permissions instead of settings permissions (#29356) --- app/workers/scheduler/auto_close_registrations_scheduler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/scheduler/auto_close_registrations_scheduler.rb b/app/workers/scheduler/auto_close_registrations_scheduler.rb index 979b336fa..f774dda11 100644 --- a/app/workers/scheduler/auto_close_registrations_scheduler.rb +++ b/app/workers/scheduler/auto_close_registrations_scheduler.rb @@ -26,7 +26,7 @@ class Scheduler::AutoCloseRegistrationsScheduler def switch_to_approval_mode! Setting.registrations_mode = 'approved' - User.those_who_can(:view_devops).includes(:account).find_each do |user| + User.those_who_can(:manage_settings).includes(:account).find_each do |user| AdminMailer.auto_close_registrations(recipient: user.account).deliver_later end end From 3fecb3673970c4495a3757cecc33c64c5af793d1 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 22 Feb 2024 18:28:41 +0100 Subject: [PATCH 05/46] Change registrations to be disabled by default for new servers (#29354) --- .../settings/registrations/show.html.haml | 2 ++ config/locales/en.yml | 1 + config/settings.yml | 2 +- spec/models/setting_spec.rb | 20 +++++-------------- spec/spec_helper.rb | 6 ++++++ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/app/views/admin/settings/registrations/show.html.haml b/app/views/admin/settings/registrations/show.html.haml index 0db9f3536..c657a7f6f 100644 --- a/app/views/admin/settings/registrations/show.html.haml +++ b/app/views/admin/settings/registrations/show.html.haml @@ -13,6 +13,8 @@ %p.lead= t('admin.settings.registrations.preamble') + .flash-message= t('admin.settings.registrations.moderation_recommandation') + .fields-row .fields-row__column.fields-row__column-6.fields-group = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: lambda { |mode| I18n.t("admin.settings.registrations_mode.modes.#{mode}") } diff --git a/config/locales/en.yml b/config/locales/en.yml index cb1ca670c..cb6ba062e 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -745,6 +745,7 @@ en: disabled: To no one users: To logged-in local users registrations: + moderation_recommandation: Please make sure you have an adequate and reactive moderation team before you open registrations to everyone! preamble: Control who can create an account on your server. title: Registrations registrations_mode: diff --git a/config/settings.yml b/config/settings.yml index f0b09dd5c..fa63fb053 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -9,7 +9,7 @@ defaults: &defaults site_terms: '' site_contact_username: '' site_contact_email: '' - registrations_mode: 'open' + registrations_mode: 'none' profile_directory: true closed_registrations_message: '' open_deletion: true diff --git a/spec/models/setting_spec.rb b/spec/models/setting_spec.rb index 3ccc21d6c..ef81b97c9 100644 --- a/spec/models/setting_spec.rb +++ b/spec/models/setting_spec.rb @@ -142,22 +142,12 @@ RSpec.describe Setting, type: :model do context 'records includes nothing' do let(:records) { [] } - context 'default_value is not a Hash' do - it 'includes Setting with value of default_value' do - setting = described_class.all_as_records[key] + it 'includes Setting with value of default_value' do + setting = described_class.all_as_records[key] - expect(setting).to be_kind_of Setting - expect(setting).to have_attributes(var: key) - expect(setting).to have_attributes(value: 'default_value') - end - end - - context 'default_value is a Hash' do - let(:default_value) { { 'foo' => 'fuga' } } - - it 'returns {}' do - expect(described_class.all_as_records).to eq({}) - end + expect(setting).to be_a described_class + expect(setting).to have_attributes(var: key) + expect(setting).to have_attributes(value: default_value) end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 0414ba9ed..afb84e193 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -30,6 +30,12 @@ RSpec.configure do |config| config.before :suite do Rails.application.load_seed Chewy.strategy(:bypass) + + # NOTE: we switched registrations mode to closed by default, but the specs + # very heavily rely on having it enabled by default, as it relies on users + # being approved by default except in select cases where explicitly testing + # other registration modes + Setting.registrations_mode = 'open' end config.after :suite do From f0541adbd44bc031fb8b070e24a605e0a0e853e4 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 22 Feb 2024 19:12:57 +0100 Subject: [PATCH 06/46] Fix link verifications when page size exceeds 1MB (#29362) --- app/services/verify_link_service.rb | 2 +- spec/services/verify_link_service_spec.rb | 27 ++++++++++++----------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb index 707aeb4e0..b317fc31a 100644 --- a/app/services/verify_link_service.rb +++ b/app/services/verify_link_service.rb @@ -19,7 +19,7 @@ class VerifyLinkService < BaseService def perform_request! @body = Request.new(:get, @url).add_headers('Accept' => 'text/html').perform do |res| - res.code == 200 ? res.body_with_limit : nil + res.code == 200 ? res.truncated_body : nil end end diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb index 8f65f3a84..0d08ec14e 100644 --- a/spec/services/verify_link_service_spec.rb +++ b/spec/services/verify_link_service_spec.rb @@ -74,6 +74,20 @@ RSpec.describe VerifyLinkService, type: :service do end context 'when a document is truncated but the link back is valid' do + let(:html) do + " + + + + " + end + + it 'marks the field as verified' do + expect(field.verified?).to be true + end + end + + context 'when a link tag might be truncated' do let(:html) do " @@ -87,19 +101,6 @@ RSpec.describe VerifyLinkService, type: :service do end end - context 'when a link back might be truncated' do - let(:html) do - " - - - Date: Fri, 23 Feb 2024 09:53:09 +0100 Subject: [PATCH 07/46] Fix processing of `Link` objects in `Image` objects (#29364) --- .../activitypub/process_account_service.rb | 11 ++-- .../process_account_service_spec.rb | 51 +++++++++++++++---- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index fef3781e1..d2223938d 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -199,10 +199,15 @@ class ActivityPub::ProcessAccountService < BaseService value = first_of_value(@json[key]) return if value.nil? - return value['url'] if value.is_a?(Hash) - image = fetch_resource_without_id_validation(value) - image['url'] if image + if value.is_a?(String) + value = fetch_resource_without_id_validation(value) + return if value.nil? + end + + value = first_of_value(value['url']) if value.is_a?(Hash) && value['type'] == 'Image' + value = value['href'] if value.is_a?(Hash) + value if value.is_a?(String) end def public_key diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 2b20d17b1..fbb5d5897 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' RSpec.describe ActivityPub::ProcessAccountService, type: :service do subject { described_class.new } - context 'property values' do + context 'with property values, an avatar, and a profile header' do let(:payload) do { id: 'https://foo.test', @@ -14,19 +14,50 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' }, { type: 'PropertyValue', name: 'non-string', value: ['foo', 'bar'] }, ], + image: { + type: 'Image', + mediaType: 'image/png', + url: 'https://foo.test/image.png', + }, + icon: { + type: 'Image', + url: [ + { + mediaType: 'image/png', + href: 'https://foo.test/icon.png', + }, + ], + }, }.with_indifferent_access end - it 'parses out of attachment' do + before do + stub_request(:get, 'https://foo.test/image.png').to_return(request_fixture('avatar.txt')) + stub_request(:get, 'https://foo.test/icon.png').to_return(request_fixture('avatar.txt')) + end + + it 'parses property values, avatar and profile header as expected' do account = subject.call('alice', 'example.com', payload) - expect(account.fields).to be_a Array - expect(account.fields.size).to eq 2 - expect(account.fields[0]).to be_a Account::Field - expect(account.fields[0].name).to eq 'Pronouns' - expect(account.fields[0].value).to eq 'They/them' - expect(account.fields[1]).to be_a Account::Field - expect(account.fields[1].name).to eq 'Occupation' - expect(account.fields[1].value).to eq 'Unit test' + + expect(account.fields) + .to be_an(Array) + .and have_attributes(size: 2) + expect(account.fields.first) + .to be_an(Account::Field) + .and have_attributes( + name: eq('Pronouns'), + value: eq('They/them') + ) + expect(account.fields.last) + .to be_an(Account::Field) + .and have_attributes( + name: eq('Occupation'), + value: eq('Unit test') + ) + expect(account).to have_attributes( + avatar_remote_url: 'https://foo.test/icon.png', + header_remote_url: 'https://foo.test/image.png' + ) end end From c2d8666bbfd8f771b759ead50a41535f988efe9c Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 23 Feb 2024 14:09:38 +0100 Subject: [PATCH 08/46] Bump version to v4.1.16 (#29371) --- CHANGELOG.md | 20 ++++++++++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57fabea9f..e65ebd8f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,26 @@ Changelog All notable changes to this project will be documented in this file. +## [4.1.16] - 2024-02-23 + +### Added + +- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355)) + In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week. + When this happens, users with the permission to change server settings will receive an email notification. + This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`. + +### Changed + +- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280)) + If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations. + Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again. + +### Fixed + +- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335)) +- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358)) + ## [4.1.15] - 2024-02-16 ### Fixed diff --git a/docker-compose.yml b/docker-compose.yml index 3e4a14413..acaeafa63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.1.15 + image: ghcr.io/mastodon/mastodon:v4.1.16 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.1.15 + image: ghcr.io/mastodon/mastodon:v4.1.16 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.1.15 + image: ghcr.io/mastodon/mastodon:v4.1.16 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 29a22eff3..d5babbe1d 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 15 + 16 end def flags From 142dd34b688d05304951e7b90ae434c903ad46fe Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 16 May 2024 11:28:04 +0200 Subject: [PATCH 09/46] Fix CI not actually running ruby tests in 4.1 branch (#30321) --- .github/workflows/test-ruby.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml index 90467e46d..fb956f3eb 100644 --- a/.github/workflows/test-ruby.yml +++ b/.github/workflows/test-ruby.yml @@ -149,3 +149,5 @@ jobs: - name: Load database schema run: './bin/rails db:create db:schema:load db:seed' + + - run: bin/rspec From 37adb144dbb0729515546eb86076b3223e9b6fc5 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 16 May 2024 11:52:02 +0200 Subject: [PATCH 10/46] Fix auto close registration mail (#30323) --- app/workers/scheduler/auto_close_registrations_scheduler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/scheduler/auto_close_registrations_scheduler.rb b/app/workers/scheduler/auto_close_registrations_scheduler.rb index f774dda11..f44c9e304 100644 --- a/app/workers/scheduler/auto_close_registrations_scheduler.rb +++ b/app/workers/scheduler/auto_close_registrations_scheduler.rb @@ -27,7 +27,7 @@ class Scheduler::AutoCloseRegistrationsScheduler Setting.registrations_mode = 'approved' User.those_who_can(:manage_settings).includes(:account).find_each do |user| - AdminMailer.auto_close_registrations(recipient: user.account).deliver_later + AdminMailer.auto_close_registrations(user.account).deliver_later end end end From 57b72cccc4e67121f7ba0239e2e4f4f2aa5fe3b4 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 14 Dec 2023 09:02:33 -0500 Subject: [PATCH 11/46] Fix reference to non-existent var in CLI maintenance command (#28363) --- lib/mastodon/maintenance_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 85937da81..f9af3c77f 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -234,7 +234,7 @@ module Mastodon users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse ref_user = users.shift @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}." - @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}" + @prompt.warn "e-mail will be disabled for the following accounts: #{users.map(&:account).map(&:acct).join(', ')}" @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.' i = 0 From 079d3e5189903c197eef439dd14a893b16ca044b Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jan 2024 15:14:46 +0100 Subject: [PATCH 12/46] Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` (#28592) --- app/controllers/well_known/webfinger_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb index f83a62a1f..6dae6a443 100644 --- a/app/controllers/well_known/webfinger_controller.rb +++ b/app/controllers/well_known/webfinger_controller.rb @@ -20,7 +20,7 @@ module WellKnown def set_account username = username_from_resource @account = begin - if username == Rails.configuration.x.local_domain + if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain Account.representative else Account.find_local!(username) From 209632a0fd585690e5c456b54f8781a7e822669a Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Tue, 6 Feb 2024 13:38:14 +0100 Subject: [PATCH 13/46] Return domain block digests from admin domain blocks API (#29092) --- app/serializers/rest/admin/domain_block_serializer.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/serializers/rest/admin/domain_block_serializer.rb b/app/serializers/rest/admin/domain_block_serializer.rb index b955d008a..e94a337cb 100644 --- a/app/serializers/rest/admin/domain_block_serializer.rb +++ b/app/serializers/rest/admin/domain_block_serializer.rb @@ -1,11 +1,15 @@ # frozen_string_literal: true class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer - attributes :id, :domain, :created_at, :severity, + attributes :id, :domain, :digest, :created_at, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate def id object.id.to_s end + + def digest + object.domain_digest + end end From affbb10566fd4388f5b761cc66857be15fd010eb Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 23 Feb 2024 20:04:57 +0100 Subject: [PATCH 14/46] Fix admin account created by `mastodon:setup` not being auto-approved (#29379) --- lib/tasks/mastodon.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 1184e5273..8c6c4be89 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -516,6 +516,7 @@ namespace :mastodon do owner_role = UserRole.find_by(name: 'Owner') user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role) user.save(validate: false) + user.approve! Setting.site_contact_username = username From 3ff575f54cd7c8156557852fb31601913cae60f3 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Mon, 11 Mar 2024 18:28:08 +0900 Subject: [PATCH 15/46] Normalize idna domain before account unblock domain (#29530) --- app/models/concerns/account_interactions.rb | 6 ++- .../concerns/account_interactions_spec.rb | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index de8bf338f..7e2e2bda4 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -187,7 +187,7 @@ module AccountInteractions end def unblock_domain!(other_domain) - block = domain_blocks.find_by(domain: other_domain) + block = domain_blocks.find_by(domain: normalized_domain(other_domain)) block&.destroy end @@ -299,4 +299,8 @@ module AccountInteractions def remove_potential_friendship(other_account) PotentialFriendshipTracker.remove(id, other_account.id) end + + def normalized_domain(domain) + TagManager.instance.normalize_domain(domain) + end end diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb index 1d1898ab0..dc7d5e972 100644 --- a/spec/models/concerns/account_interactions_spec.rb +++ b/spec/models/concerns/account_interactions_spec.rb @@ -248,6 +248,24 @@ describe AccountInteractions do end end + describe '#block_idna_domain!' do + subject do + [ + account.block_domain!(idna_domain), + account.block_domain!(punycode_domain), + ] + end + + let(:idna_domain) { '대한민국.한국' } + let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' } + + it 'creates single AccountDomainBlock' do + expect do + expect(subject).to all(be_a AccountDomainBlock) + end.to change { account.domain_blocks.count }.by 1 + end + end + describe '#unfollow!' do subject { account.unfollow!(target_account) } @@ -343,6 +361,28 @@ describe AccountInteractions do end end + describe '#unblock_idna_domain!' do + subject { account.unblock_domain!(punycode_domain) } + + let(:idna_domain) { '대한민국.한국' } + let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' } + + context 'when blocking the domain' do + it 'returns destroyed AccountDomainBlock' do + account_domain_block = Fabricate(:account_domain_block, domain: idna_domain) + account.domain_blocks << account_domain_block + expect(subject).to be_a AccountDomainBlock + expect(subject).to be_destroyed + end + end + + context 'when unblocking idna domain' do + it 'returns nil' do + expect(subject).to be_nil + end + end + end + describe '#following?' do subject { account.following?(target_account) } From 8ce403a85b5e79ed8b7a429397092f5efb137b67 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Fri, 22 Mar 2024 11:08:27 -0400 Subject: [PATCH 16/46] Fix results/query in `api/v1/featured_tags/suggestions` (#29597) --- .../featured_tags/suggestions_controller.rb | 6 ++- .../suggestions_controller_spec.rb | 45 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb diff --git a/app/controllers/api/v1/featured_tags/suggestions_controller.rb b/app/controllers/api/v1/featured_tags/suggestions_controller.rb index 76633210a..4f732ed2d 100644 --- a/app/controllers/api/v1/featured_tags/suggestions_controller.rb +++ b/app/controllers/api/v1/featured_tags/suggestions_controller.rb @@ -12,6 +12,10 @@ class Api::V1::FeaturedTags::SuggestionsController < Api::BaseController private def set_recently_used_tags - @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10) + @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10) + end + + def featured_tag_ids + current_account.featured_tags.pluck(:tag_id) end end diff --git a/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb new file mode 100644 index 000000000..8cb928ea2 --- /dev/null +++ b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::FeaturedTags::SuggestionsController do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:accounts') } + let(:account) { Fabricate(:account, user: user) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let!(:unused_featured_tag) { Fabricate(:tag, name: 'unused_featured_tag') } + let!(:used_tag) { Fabricate(:tag, name: 'used_tag') } + let!(:used_featured_tag) { Fabricate(:tag, name: 'used_featured_tag') } + + before do + _unused_tag = Fabricate(:tag, name: 'unused_tag') + + # Make relevant tags used by account + status = Fabricate(:status, account: account) + status.tags << used_tag + status.tags << used_featured_tag + + # Feature the relevant tags + Fabricate :featured_tag, account: account, name: unused_featured_tag.name + Fabricate :featured_tag, account: account, name: used_featured_tag.name + end + + it 'returns http success and recently used but not featured tags', :aggregate_failures do + get :index, params: { account_id: account.id, limit: 2 } + + expect(response) + .to have_http_status(200) + expect(body_as_json) + .to contain_exactly( + include(name: used_tag.name) + ) + end + end +end From ba5551fd1d6eae3edb45b2ee3934e2d3c5c781ae Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 5 Apr 2024 09:48:45 +0200 Subject: [PATCH 17/46] Improve email address validation (#29838) --- Gemfile | 1 + Gemfile.lock | 1 + app/models/user.rb | 3 +++ app/validators/email_address_validator.rb | 18 ++++++++++++++++++ spec/models/user_spec.rb | 6 ++++++ 5 files changed, 29 insertions(+) create mode 100644 app/validators/email_address_validator.rb diff --git a/Gemfile b/Gemfile index 6e5292d82..f5b43934a 100644 --- a/Gemfile +++ b/Gemfile @@ -158,3 +158,4 @@ gem 'concurrent-ruby', require: false gem 'connection_pool', require: false gem 'xorcist', '~> 1.1' gem 'cocoon', '~> 1.2' +gem 'mail', '~> 2.8' diff --git a/Gemfile.lock b/Gemfile.lock index 29a437fb0..e43345727 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -816,6 +816,7 @@ DEPENDENCIES letter_opener_web (~> 2.0) link_header (~> 0.0) lograge (~> 0.12) + mail (~> 2.8) makara (~> 0.5) mario-redis-lock (~> 1.2) memory_profiler diff --git a/app/models/user.rb b/app/models/user.rb index 810ced84a..e23559155 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -94,6 +94,9 @@ class User < ApplicationRecord validates :invite_request, presence: true, on: :create, if: :invite_text_required? validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale? + + validates :email, presence: true, email_address: true + validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? } validates_with EmailMxValidator, if: :validate_email_dns? validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create diff --git a/app/validators/email_address_validator.rb b/app/validators/email_address_validator.rb new file mode 100644 index 000000000..ed0bb1165 --- /dev/null +++ b/app/validators/email_address_validator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing +# with an indirect dependency of ours, `validate_email`, which, turns out, +# has the same approach as we do, but with an extra check disallowing +# single-label domains. Decided to not switch to `validate_email` because +# we do want to allow at least `localhost`. + +class EmailAddressValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + value = value.strip + + address = Mail::Address.new(value) + record.errors.add(attribute, :invalid) if address.address != value + rescue Mail::Field::FieldError + record.errors.add(attribute, :invalid) + end +end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 6414917a0..79cd43f6d 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -40,6 +40,12 @@ RSpec.describe User, type: :model do expect(user.valid?).to be true end + it 'is valid with a localhost e-mail address' do + user = Fabricate.build(:user, email: 'admin@localhost') + user.valid? + expect(user.valid?).to be true + end + it 'cleans out empty string from languages' do user = Fabricate.build(:user, chosen_languages: ['']) user.valid? From 5973d7a4b628c2f9cfe2c8dd4c0f29765b633301 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 8 Apr 2024 15:46:13 +0200 Subject: [PATCH 18/46] Remove caching in `cache_collection` (#29862) --- app/controllers/concerns/cache_concern.rb | 24 ++++---------- app/models/concerns/cacheable.rb | 4 +++ app/models/feed.rb | 2 +- app/models/public_feed.rb | 2 +- app/models/status.rb | 32 ------------------ app/models/tag_feed.rb | 2 +- .../application_controller_spec.rb | 33 ------------------- spec/models/home_feed_spec.rb | 1 - 8 files changed, 14 insertions(+), 86 deletions(-) diff --git a/app/controllers/concerns/cache_concern.rb b/app/controllers/concerns/cache_concern.rb index 05e431b19..d12efe3a9 100644 --- a/app/controllers/concerns/cache_concern.rb +++ b/app/controllers/concerns/cache_concern.rb @@ -28,29 +28,19 @@ module CacheConcern response.headers['Vary'] = public_fetch_mode? ? 'Accept' : 'Accept, Signature' end + # TODO: Rename this method, as it does not perform any caching anymore. def cache_collection(raw, klass) - return raw unless klass.respond_to?(:with_includes) + return raw unless klass.respond_to?(:preload_cacheable_associations) - raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation) - return [] if raw.empty? + records = raw.to_a - cached_keys_with_value = Rails.cache.read_multi(*raw).transform_keys(&:id) - uncached_ids = raw.map(&:id) - cached_keys_with_value.keys + klass.preload_cacheable_associations(records) - klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!) - - unless uncached_ids.empty? - uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id) - - uncached.each_value do |item| - Rails.cache.write(item, item) - end - end - - raw.filter_map { |item| cached_keys_with_value[item.id] || uncached[item.id] } + records end + # TODO: Rename this method, as it does not perform any caching anymore. def cache_collection_paginated_by_id(raw, klass, limit, options) - cache_collection raw.cache_ids.to_a_paginated_by_id(limit, options), klass + cache_collection raw.to_a_paginated_by_id(limit, options), klass end end diff --git a/app/models/concerns/cacheable.rb b/app/models/concerns/cacheable.rb index d7524cdfd..0d0826232 100644 --- a/app/models/concerns/cacheable.rb +++ b/app/models/concerns/cacheable.rb @@ -14,6 +14,10 @@ module Cacheable includes(@cache_associated) end + def preload_cacheable_associations(records) + ActiveRecord::Associations::Preloader.new.preload(records, @cache_associated) + end + def cache_ids select(:id, :updated_at) end diff --git a/app/models/feed.rb b/app/models/feed.rb index f51dcfab1..30073fed4 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -28,7 +28,7 @@ class Feed unhydrated = redis.zrangebyscore(key, "(#{min_id}", "(#{max_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i) end - Status.where(id: unhydrated).cache_ids + Status.where(id: unhydrated) end def key diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb index 1cfd9a500..e88912fb6 100644 --- a/app/models/public_feed.rb +++ b/app/models/public_feed.rb @@ -29,7 +29,7 @@ class PublicFeed scope.merge!(media_only_scope) if media_only? scope.merge!(language_scope) if account&.chosen_languages.present? - scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) + scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end private diff --git a/app/models/status.rb b/app/models/status.rb index afd3747ea..0ae22430c 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -344,38 +344,6 @@ class Status < ApplicationRecord StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |p, h| h[p.status_id] = true } end - def reload_stale_associations!(cached_items) - account_ids = [] - - cached_items.each do |item| - account_ids << item.account_id - account_ids << item.reblog.account_id if item.reblog? - end - - account_ids.uniq! - - status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq - - return if account_ids.empty? - - accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id) - - status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id) - - cached_items.each do |item| - item.account = accounts[item.account_id] - item.reblog.account = accounts[item.reblog.account_id] if item.reblog? - - if item.reblog? - status_stat = status_stats[item.reblog.id] - item.reblog.status_stat = status_stat if status_stat.present? - else - status_stat = status_stats[item.id] - item.status_stat = status_stat if status_stat.present? - end - end - end - def from_text(text) return [] if text.blank? diff --git a/app/models/tag_feed.rb b/app/models/tag_feed.rb index b8cd63557..6b5831d24 100644 --- a/app/models/tag_feed.rb +++ b/app/models/tag_feed.rb @@ -33,7 +33,7 @@ class TagFeed < PublicFeed scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? - scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) + scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end private diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index 1b002e01c..bfab2818e 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -242,37 +242,4 @@ describe ApplicationController, type: :controller do include_examples 'respond_with_error', 422 end - - describe 'cache_collection' do - class C < ApplicationController - public :cache_collection - end - - shared_examples 'receives :with_includes' do |fabricator, klass| - it 'uses raw if it is not an ActiveRecord::Relation' do - record = Fabricate(fabricator) - expect(C.new.cache_collection([record], klass)).to eq [record] - end - end - - shared_examples 'cacheable' do |fabricator, klass| - include_examples 'receives :with_includes', fabricator, klass - - it 'calls cache_ids of raw if it is an ActiveRecord::Relation' do - record = Fabricate(fabricator) - relation = klass.none - allow(relation).to receive(:cache_ids).and_return([record]) - expect(C.new.cache_collection(relation, klass)).to eq [record] - end - end - - it 'returns raw unless class responds to :with_includes' do - raw = Object.new - expect(C.new.cache_collection(raw, Object)).to eq raw - end - - context 'Status' do - include_examples 'cacheable', :status, Status - end - end end diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb index 80f6edbff..cdc3f5ad4 100644 --- a/spec/models/home_feed_spec.rb +++ b/spec/models/home_feed_spec.rb @@ -25,7 +25,6 @@ RSpec.describe HomeFeed, type: :model do results = subject.get(3) expect(results.map(&:id)).to eq [3, 2] - expect(results.first.attributes.keys).to eq %w(id updated_at) end end From a6089cdfca1d8089318dd694edf44db8ce9bec59 Mon Sep 17 00:00:00 2001 From: Tim Rogers Date: Mon, 22 Apr 2024 04:00:24 -0500 Subject: [PATCH 19/46] Fixed crash when supplying FFMPEG_BINARY environment variable (#30022) --- app/lib/video_metadata_extractor.rb | 2 +- config/initializers/ffmpeg.rb | 7 +++++-- lib/paperclip/image_extractor.rb | 2 +- lib/paperclip/transcoder.rb | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index f27d34868..df5409375 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -22,7 +22,7 @@ class VideoMetadataExtractor private def ffmpeg_command_output - command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') + command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') command.run(path: @path, format: 'json', loglevel: 'fatal') end diff --git a/config/initializers/ffmpeg.rb b/config/initializers/ffmpeg.rb index 4c0bf779d..87f85eeec 100644 --- a/config/initializers/ffmpeg.rb +++ b/config/initializers/ffmpeg.rb @@ -1,3 +1,6 @@ -if ENV['FFMPEG_BINARY'].present? - FFMPEG.ffmpeg_binary = ENV['FFMPEG_BINARY'] +# frozen_string_literal: true + +Rails.application.configure do + config.x.ffmpeg_binary = ENV['FFMPEG_BINARY'] || 'ffmpeg' + config.x.ffprobe_binary = ENV['FFPROBE_BINARY'] || 'ffprobe' end diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index 17fe4326f..8a565d046 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -35,7 +35,7 @@ module Paperclip dst.binmode begin - command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) + command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) command.run(source: @file.path, destination: dst.path, loglevel: 'fatal') rescue Terrapin::ExitStatusError dst.close(true) diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index 0f2e30f7d..e7cf78f8c 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -51,7 +51,7 @@ module Paperclip command_arguments, interpolations = prepare_command(destination) begin - command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger) + command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, command_arguments.join(' '), logger: Paperclip.logger) command.run(interpolations) rescue Terrapin::ExitStatusError => e raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}" From 70c4d70dbedb94ecde099cd71bf7a4deb01d8598 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 26 Apr 2024 15:19:02 +0200 Subject: [PATCH 20/46] Fix Idempotency-Key ignored when scheduling a post (#30084) --- app/services/post_status_service.rb | 4 ++-- spec/services/post_status_service_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index bd3b69632..7b57c05f5 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -137,7 +137,7 @@ class PostStatusService < BaseService def idempotency_duplicate if scheduled? - @account.schedule_statuses.find(@idempotency_duplicate) + @account.scheduled_statuses.find(@idempotency_duplicate) else @account.statuses.find(@idempotency_duplicate) end @@ -189,7 +189,7 @@ class PostStatusService < BaseService end def scheduled_options - @options.tap do |options_hash| + @options.dup.tap do |options_hash| options_hash[:in_reply_to_id] = options_hash.delete(:thread)&.id options_hash[:application_id] = options_hash.delete(:application)&.id options_hash[:scheduled_at] = nil diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index d21270c79..c56f9efdb 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -52,6 +52,13 @@ RSpec.describe PostStatusService, type: :service do it 'does not change statuses count' do expect { subject.call(account, text: 'Hi future!', scheduled_at: future, thread: previous_status) }.not_to change { [account.statuses_count, previous_status.replies_count] } end + + it 'returns existing status when used twice with idempotency key' do + account = Fabricate(:account) + status1 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future) + status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future) + expect(status2.id).to eq status1.id + end end it 'creates response to the original status of boost' do From 33a50884e5ea30bb74eafedfe78a2885f22eed1e Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 2 May 2024 22:56:21 +0200 Subject: [PATCH 21/46] Fix not being able to block a subdomain of an already-blocked domain through the API (#30119) --- .../api/v1/admin/domain_blocks_controller.rb | 9 ++- .../v1/admin/domain_blocks_controller_spec.rb | 71 ++++++++++++++----- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/v1/admin/domain_blocks_controller.rb b/app/controllers/api/v1/admin/domain_blocks_controller.rb index 8b77e9717..dacab4d19 100644 --- a/app/controllers/api/v1/admin/domain_blocks_controller.rb +++ b/app/controllers/api/v1/admin/domain_blocks_controller.rb @@ -19,10 +19,11 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController def create authorize :domain_block, :create? + @domain_block = DomainBlock.new(resource_params) existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil - return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if existing_domain_block.present? + return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if conflicts_with_existing_block?(@domain_block, existing_domain_block) - @domain_block = DomainBlock.create!(resource_params) + @domain_block.save! DomainBlockWorker.perform_async(@domain_block.id) log_action :create, @domain_block render json: @domain_block, serializer: REST::Admin::DomainBlockSerializer @@ -55,6 +56,10 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController private + def conflicts_with_existing_block?(domain_block, existing_domain_block) + existing_domain_block.present? && (existing_domain_block.domain == TagManager.instance.normalize_domain(domain_block.domain) || !domain_block.stricter_than?(existing_domain_block)) + end + def set_domain_blocks @domain_blocks = filtered_domain_blocks.order(id: :desc).to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id)) end diff --git a/spec/controllers/api/v1/admin/domain_blocks_controller_spec.rb b/spec/controllers/api/v1/admin/domain_blocks_controller_spec.rb index 606def602..dc65aef5d 100644 --- a/spec/controllers/api/v1/admin/domain_blocks_controller_spec.rb +++ b/spec/controllers/api/v1/admin/domain_blocks_controller_spec.rb @@ -16,6 +16,8 @@ RSpec.describe Api::V1::Admin::DomainBlocksController, type: :controller do let(:scopes) { wrong_scope } it 'returns http forbidden' do + subject + expect(response).to have_http_status(403) end end @@ -24,6 +26,8 @@ RSpec.describe Api::V1::Admin::DomainBlocksController, type: :controller do let(:role) { UserRole.find_by(name: wrong_role) } it 'returns http forbidden' do + subject + expect(response).to have_http_status(403) end end @@ -140,39 +144,70 @@ RSpec.describe Api::V1::Admin::DomainBlocksController, type: :controller do describe 'POST #create' do let(:existing_block_domain) { 'example.com' } + let(:params) { { domain: 'foo.bar.com', severity: :silence } } let!(:block) { Fabricate(:domain_block, domain: existing_block_domain, severity: :suspend) } - before do - post :create, params: { domain: 'foo.bar.com', severity: :silence } + subject do + post :create, params: params end it_behaves_like 'forbidden for wrong scope', 'write:statuses' it_behaves_like 'forbidden for wrong role', '' it_behaves_like 'forbidden for wrong role', 'Moderator' - it 'returns http success' do + it 'creates a domain block and returns expected domain name', :aggregate_failures do + subject + expect(response).to have_http_status(200) - end - - it 'returns expected domain name' do - json = body_as_json - expect(json[:domain]).to eq 'foo.bar.com' - end - - it 'creates a domain block' do + expect(body_as_json[:domain]).to eq 'foo.bar.com' expect(DomainBlock.find_by(domain: 'foo.bar.com')).to_not be_nil end - context 'when a stricter domain block already exists' do - let(:existing_block_domain) { 'bar.com' } + context 'when a looser domain block already exists on a higher level domain' do + let(:params) { { domain: 'foo.bar.com', severity: :suspend } } - it 'returns http unprocessable entity' do - expect(response).to have_http_status(422) + before do + Fabricate(:domain_block, domain: 'bar.com', severity: :silence) end - it 'renders existing domain block in error' do - json = body_as_json - expect(json[:existing_domain_block][:domain]).to eq existing_block_domain + it 'creates a domain block with the expected domain name and severity', :aggregate_failures do + subject + + body = body_as_json + + expect(response).to have_http_status(200) + expect(body).to match a_hash_including( + { + domain: 'foo.bar.com', + severity: 'suspend', + } + ) + + expect(DomainBlock.find_by(domain: 'foo.bar.com')).to be_present + end + end + + context 'when a domain block already exists on the same domain' do + before do + Fabricate(:domain_block, domain: 'foo.bar.com', severity: :silence) + end + + it 'returns existing domain block in error', :aggregate_failures do + subject + + expect(response).to have_http_status(422) + expect(body_as_json[:existing_domain_block][:domain]).to eq('foo.bar.com') + end + end + + context 'when a stricter domain block already exists on a higher level domain' do + let(:existing_block_domain) { 'bar.com' } + + it 'returns http unprocessable entity with existing domain block in error', :aggregate_reblogs do + subject + + expect(response).to have_http_status(422) + expect(body_as_json[:existing_domain_block][:domain]).to eq existing_block_domain end end end From 984d7d3dc8c8fd11d5469e9169eeee12c4c46a64 Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Tue, 30 Apr 2024 10:48:02 +0200 Subject: [PATCH 22/46] Fix missing destory audit logs for Domain Allows (#30125) --- app/controllers/admin/domain_allows_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb index 31be1978b..b0f139e3a 100644 --- a/app/controllers/admin/domain_allows_controller.rb +++ b/app/controllers/admin/domain_allows_controller.rb @@ -25,6 +25,8 @@ class Admin::DomainAllowsController < Admin::BaseController def destroy authorize @domain_allow, :destroy? UnallowDomainService.new.call(@domain_allow) + log_action :destroy, @domain_allow + redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg') end From 3aec33f5a2c2008ca7a1fae24777b9598f9ee45d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 May 2024 15:11:13 +0200 Subject: [PATCH 23/46] Fix off-by-one in `tootctl media` commands (#30306) --- lib/mastodon/media_cli.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 24cc98964..3fe5bf420 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -143,7 +143,7 @@ module Mastodon model_name = path_segments.first.classify attachment_name = path_segments[1].singularize - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i file_name = path_segments.last record = record_map.dig(model_name, record_id) attachment = record&.public_send(attachment_name) @@ -186,7 +186,7 @@ module Mastodon end model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i attachment_name = path_segments[1].singularize file_name = path_segments.last @@ -322,7 +322,7 @@ module Mastodon end model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i unless PRELOAD_MODEL_WHITELIST.include?(model_name) say("Cannot find corresponding model: #{model_name}", :red) @@ -372,7 +372,7 @@ module Mastodon next unless [7, 10].include?(segments.size) model_name = segments.first.classify - record_id = segments[2..-2].join.to_i + record_id = segments[2...-2].join.to_i next unless PRELOAD_MODEL_WHITELIST.include?(model_name) From 13bbcdf4d4937df9fe798d0a29a2527761e64acf Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 16 May 2024 10:44:27 +0200 Subject: [PATCH 24/46] Update dependency json-jwt to 1.15.3.1 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e43345727..aca77d709 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -330,7 +330,7 @@ GEM jmespath (1.6.2) json (2.6.3) json-canonicalization (0.3.0) - json-jwt (1.15.3) + json-jwt (1.15.3.1) activesupport (>= 4.2) aes_key_wrap bindata From c717b7da998070d56f01993ceaae30fdc839e56a Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 16 May 2024 10:46:04 +0200 Subject: [PATCH 25/46] Update dependency puma to 5.6.8 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index aca77d709..4e7a430b8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -491,7 +491,7 @@ GEM pry-rails (0.3.9) pry (>= 0.10.4) public_suffix (5.0.1) - puma (5.6.7) + puma (5.6.8) nio4r (~> 2.0) pundit (2.3.0) activesupport (>= 3.0.0) From b39fbe7c833a0ff23776b03c31b6815d8fe688d7 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 16 May 2024 10:47:45 +0200 Subject: [PATCH 26/46] Update dependency nokogiri to 1.16.5 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4e7a430b8..20e595d95 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -424,7 +424,7 @@ GEM net-protocol net-ssh (7.0.1) nio4r (2.5.9) - nokogiri (1.16.2) + nokogiri (1.16.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.2.8) From b4e3a789b1b5f81f4e651a8c2c338e399c739d3d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 29 May 2024 11:17:20 +0200 Subject: [PATCH 27/46] Update dependency rexml to 3.2.8 --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 20e595d95..6ac2df615 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -565,7 +565,8 @@ GEM responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) - rexml (3.2.5) + rexml (3.2.8) + strscan (>= 3.0.9) rotp (6.2.0) rpam2 (4.0.2) rqrcode (2.1.2) @@ -679,6 +680,7 @@ GEM redlock (~> 1.0) strong_migrations (0.7.9) activerecord (>= 5) + strscan (3.1.0) swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) From 257f9abd5606504a1a258aacaabc7963c142dd11 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 29 May 2024 10:15:06 +0200 Subject: [PATCH 28/46] Fix leaking Elasticsearch connections in Sidekiq processes (#30450) --- lib/mastodon/sidekiq_middleware.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index 9832e1a27..c6bfa492f 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -8,6 +8,7 @@ class Mastodon::SidekiqMiddleware rescue Mastodon::HostValidationError # Do not retry rescue => e + clean_up_elasticsearch_connections! limit_backtrace_and_raise(e) ensure clean_up_sockets! @@ -25,6 +26,32 @@ class Mastodon::SidekiqMiddleware clean_up_statsd_socket! end + # This is a hack to immediately free up unused Elasticsearch connections. + # + # Indeed, Chewy creates one `Elasticsearch::Client` instance per thread, + # and each such client manages its long-lasting connection to + # Elasticsearch. + # + # As far as I know, neither `chewy`, `elasticsearch-transport` or even + # `faraday` provide a reliable way to immediately close a connection, and + # rely on the underlying object to be garbage-collected instead. + # + # Furthermore, `sidekiq` creates a new thread each time a job throws an + # exception, meaning that each failure will create a new connection, and + # the old one will only be closed on full garbage collection. + def clean_up_elasticsearch_connections! + return unless Chewy.enabled? && Chewy.current[:chewy_client].present? + + Chewy.client.transport.connections.each do |connection| + # NOTE: This bit of code is tailored for the HTTPClient Faraday adapter + connection.connection.app.instance_variable_get(:@client)&.reset_all + end + + Chewy.current.delete(:chewy_client) + rescue + nil + end + def clean_up_redis_socket! RedisConfiguration.pool.checkin if Thread.current[:redis] Thread.current[:redis] = nil From ba240cea0c935cc8ca095b4bf91600c1fa82c68a Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 23 May 2024 19:28:18 +0200 Subject: [PATCH 29/46] Normalize language code of incoming posts (#30403) --- app/lib/activitypub/parser/status_parser.rb | 11 +++- .../activitypub/parser/status_parser_spec.rb | 50 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 spec/lib/activitypub/parser/status_parser_spec.rb diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb index 45f5fc5bf..6699dc21b 100644 --- a/app/lib/activitypub/parser/status_parser.rb +++ b/app/lib/activitypub/parser/status_parser.rb @@ -3,6 +3,8 @@ class ActivityPub::Parser::StatusParser include JsonLdHelper + NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze + # @param [Hash] json # @param [Hash] magic_values # @option magic_values [String] :followers_collection @@ -86,6 +88,13 @@ class ActivityPub::Parser::StatusParser end def language + lang = raw_language_code + lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang) + end + + private + + def raw_language_code if content_language_map? @object['contentMap'].keys.first elsif name_language_map? @@ -95,8 +104,6 @@ class ActivityPub::Parser::StatusParser end end - private - def audience_to as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } end diff --git a/spec/lib/activitypub/parser/status_parser_spec.rb b/spec/lib/activitypub/parser/status_parser_spec.rb new file mode 100644 index 000000000..5d9f008db --- /dev/null +++ b/spec/lib/activitypub/parser/status_parser_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ActivityPub::Parser::StatusParser do + subject { described_class.new(json) } + + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') } + let(:follower) { Fabricate(:account, username: 'bob') } + + let(:json) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: [ActivityPub::TagManager.instance.uri_for(sender), '#foo'].join, + type: 'Create', + actor: ActivityPub::TagManager.instance.uri_for(sender), + object: object_json, + }.with_indifferent_access + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'), + type: 'Note', + to: [ + 'https://www.w3.org/ns/activitystreams#Public', + ActivityPub::TagManager.instance.uri_for(follower), + ], + content: '@bob lorem ipsum', + contentMap: { + EN: '@bob lorem ipsum', + }, + published: 1.hour.ago.utc.iso8601, + updated: 1.hour.ago.utc.iso8601, + tag: { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(follower), + }, + } + end + + it 'correctly parses status' do + expect(subject).to have_attributes( + text: '@bob lorem ipsum', + uri: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'), + reply: false, + language: :en + ) + end +end From e292a28933c390745bbce78cdc87d69c7db82354 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:03:13 +0200 Subject: [PATCH 30/46] Merge pull request from GHSA-5fq7-3p3j-9vrf --- app/services/notify_service.rb | 17 +++++++++-------- spec/services/notify_service_spec.rb | 10 +++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index c7454fc60..a0dd9bed3 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -62,16 +62,17 @@ class NotifyService < BaseService LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id WHERE s.id = :id UNION ALL - SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1 - FROM ancestors st - JOIN statuses s ON s.id = st.in_reply_to_id - LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id - WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit + SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1 + FROM ancestors + JOIN statuses s ON s.id = ancestors.in_reply_to_id + /* early exit if we already have a mention matching our requirements */ + LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id + WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit ) SELECT COUNT(*) - FROM ancestors st - JOIN statuses s ON s.id = st.id - WHERE st.mention_id IS NOT NULL AND s.visibility = 3 + FROM ancestors + JOIN statuses s ON s.id = ancestors.id + WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3 SQL end diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index 294c31b04..82770d4fe 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -72,11 +72,11 @@ RSpec.describe NotifyService, type: :service do end end - context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do - let(:reply_to) { Fabricate(:status, account: recipient) } - let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } - let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) } - let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } + context 'when the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do + let(:public_status) { Fabricate(:status, account: recipient) } + let(:intermediate_reply) { Fabricate(:status, account: sender, thread: public_status, visibility: :direct) } + let!(:intermediate_mention) { Fabricate(:mention, account: sender, status: intermediate_reply) } + let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: intermediate_reply)) } it 'does not notify' do expect { subject }.to_not change(Notification, :count) From 020228ddba50c5da461093766c0a679382c48a59 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:14:04 +0200 Subject: [PATCH 31/46] Merge pull request from GHSA-q3rg-xx5v-4mxh --- config/initializers/rack_attack.rb | 10 ++++- spec/config/initializers/rack_attack_spec.rb | 40 +++++++++++++++----- spec/rails_helper.rb | 1 + 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 3857e3055..a37e923f1 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -30,13 +30,17 @@ class Rack::Attack end def authenticated_user_id - authenticated_token&.resource_owner_id + authenticated_token&.resource_owner_id || warden_user_id end def authenticated_token_id authenticated_token&.id end + def warden_user_id + @env['warden']&.user&.id + end + def unauthenticated? !authenticated_user_id end @@ -137,6 +141,10 @@ class Rack::Attack req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in') end + throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| + req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + end + self.throttled_responder = lambda do |request| now = Time.now.utc match_data = request.env['rack.attack.match_data'] diff --git a/spec/config/initializers/rack_attack_spec.rb b/spec/config/initializers/rack_attack_spec.rb index 581021cb9..50e5d5052 100644 --- a/spec/config/initializers/rack_attack_spec.rb +++ b/spec/config/initializers/rack_attack_spec.rb @@ -1,8 +1,6 @@ require 'rails_helper' -describe Rack::Attack do - include Rack::Test::Methods - +describe Rack::Attack, type: :request do def app Rails.application end @@ -12,7 +10,7 @@ describe Rack::Attack do it 'does not change the request status' do limit.times do request.call - expect(last_response.status).to_not eq(429) + expect(response.status).to_not eq(429) end end end @@ -21,7 +19,7 @@ describe Rack::Attack do it 'returns http too many requests' do (limit * 2).times do |i| request.call - expect(last_response.status).to eq(429) if i > limit + expect(response.status).to eq(429) if i > limit end end end @@ -32,7 +30,7 @@ describe Rack::Attack do describe 'throttle excessive sign-up requests by IP address' do context 'through the website' do let(:limit) { 25 } - let(:request) { ->() { post path, {}, 'REMOTE_ADDR' => remote_ip } } + let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } } context 'for exact path' do let(:path) { '/auth' } @@ -47,7 +45,7 @@ describe Rack::Attack do context 'through the API' do let(:limit) { 5 } - let(:request) { ->() { post path, {}, 'REMOTE_ADDR' => remote_ip } } + let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } } context 'for exact path' do let(:path) { '/api/v1/accounts' } @@ -59,7 +57,7 @@ describe Rack::Attack do it 'returns http not found' do request.call - expect(last_response.status).to eq(404) + expect(response.status).to eq(404) end end end @@ -67,7 +65,7 @@ describe Rack::Attack do describe 'throttle excessive sign-in requests by IP address' do let(:limit) { 25 } - let(:request) { ->() { post path, {}, 'REMOTE_ADDR' => remote_ip } } + let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } } context 'for exact path' do let(:path) { '/auth/sign_in' } @@ -79,4 +77,28 @@ describe Rack::Attack do it_behaves_like 'throttled endpoint' end end + + describe 'throttle excessive password change requests by account' do + let(:user) { Fabricate(:user, email: 'user@host.example') } + let(:limit) { 10 } + let(:period) { 10.minutes } + let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } } + let(:path) { '/auth' } + + before do + sign_in user, scope: :user + + # Unfortunately, devise's `sign_in` helper causes the `session` to be + # loaded in the next request regardless of whether it's actually accessed + # by the client code. + # + # So, we make an extra query to clear issue a session cookie instead. + # + # A less resource-intensive way to deal with that would be to generate the + # session cookie manually, but this seems pretty involved. + get '/' + end + + it_behaves_like 'throttled endpoint' + end end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index c197bc007..759303e39 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -41,6 +41,7 @@ RSpec.configure do |config| config.include Devise::Test::ControllerHelpers, type: :controller config.include Devise::Test::ControllerHelpers, type: :view + config.include Devise::Test::IntegrationHelpers, type: :request config.include Paperclip::Shoulda::Matchers config.include ActiveSupport::Testing::TimeHelpers config.include Redisable From d770b61a749569cc222eb598e1a5d12ee8ceefe9 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:24:29 +0200 Subject: [PATCH 32/46] Merge pull request from GHSA-c2r5-cfqr-c553 * Add hardening monkey-patch to prevent IP spoofing on misconfigured installations * Remove rack-attack safelist --- config/application.rb | 1 + config/initializers/rack_attack.rb | 4 -- lib/action_dispatch/remote_ip_extensions.rb | 72 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 lib/action_dispatch/remote_ip_extensions.rb diff --git a/config/application.rb b/config/application.rb index 55d98ca99..36e73aa15 100644 --- a/config/application.rb +++ b/config/application.rb @@ -44,6 +44,7 @@ require_relative '../lib/chewy/strategy/bypass_with_warning' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' +require_relative '../lib/action_dispatch/remote_ip_extensions' require_relative '../lib/active_record/database_tasks_extensions' require_relative '../lib/active_record/batches' require_relative '../lib/simple_navigation/item_extensions' diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index a37e923f1..a3e0e3bbe 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -62,10 +62,6 @@ class Rack::Attack end end - Rack::Attack.safelist('allow from localhost') do |req| - req.remote_ip == '127.0.0.1' || req.remote_ip == '::1' - end - Rack::Attack.blocklist('deny from blocklist') do |req| IpBlock.blocked?(req.remote_ip) end diff --git a/lib/action_dispatch/remote_ip_extensions.rb b/lib/action_dispatch/remote_ip_extensions.rb new file mode 100644 index 000000000..e5c48bf3c --- /dev/null +++ b/lib/action_dispatch/remote_ip_extensions.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# Mastodon is not made to be directly accessed without a reverse proxy. +# This monkey-patch prevents remote IP address spoofing when being accessed +# directly. +# +# See PR: https://github.com/rails/rails/pull/51610 + +# In addition to the PR above, it also raises an error if a request with +# `X-Forwarded-For` or `Client-Ip` comes directly from a client without +# going through a trusted proxy. + +# rubocop:disable all -- This is a mostly vendored file + +module ActionDispatch + class RemoteIp + module GetIpExtensions + def calculate_ip + # Set by the Rack web server, this is a single value. + remote_addr = ips_from(@req.remote_addr).last + + # Could be a CSV list and/or repeated headers that were concatenated. + client_ips = ips_from(@req.client_ip).reverse! + forwarded_ips = ips_from(@req.x_forwarded_for).reverse! + + # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they + # are both set, it means that either: + # + # 1) This request passed through two proxies with incompatible IP header + # conventions. + # + # 2) The client passed one of `Client-Ip` or `X-Forwarded-For` + # (whichever the proxy servers weren't using) themselves. + # + # Either way, there is no way for us to determine which header is the right one + # after the fact. Since we have no idea, if we are concerned about IP spoofing + # we need to give up and explode. (If you're not concerned about IP spoofing you + # can turn the `ip_spoofing_check` option off.) + should_check_ip = @check_ip && client_ips.last && forwarded_ips.last + if should_check_ip && !forwarded_ips.include?(client_ips.last) + # We don't know which came from the proxy, and which from the user + raise IpSpoofAttackError, "IP spoofing attack?! " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client + if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr } + raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # We assume these things about the IP headers: + # + # - X-Forwarded-For will be a list of IPs, one per proxy, or blank + # - Client-Ip is propagated from the outermost proxy, or is blank + # - REMOTE_ADDR will be the IP that made the request to Rack + ips = forwarded_ips + client_ips + ips.compact! + + # If every single IP option is in the trusted list, return the IP that's + # furthest away + filter_proxies([remote_addr] + ips).first || ips.last || remote_addr + end + end + end +end + +ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions) + +# rubocop:enable all From 95ebcff98e9ec7f3079c4fb21b74bfc9893c1321 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 14:56:18 +0200 Subject: [PATCH 33/46] Fix rate-limiting incorrectly triggering a session cookie on most endpoints (#30483) --- config/initializers/rack_attack.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index a3e0e3bbe..e60f71985 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -30,7 +30,7 @@ class Rack::Attack end def authenticated_user_id - authenticated_token&.resource_owner_id || warden_user_id + authenticated_token&.resource_owner_id end def authenticated_token_id @@ -138,7 +138,7 @@ class Rack::Attack end throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| - req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) + req.warden_user_id if req.put? || (req.patch? && req.path_matches?('/auth')) end self.throttled_responder = lambda do |request| From 1cad857f14f08370a023e46c24eb81f702bc7e86 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 May 2024 15:49:14 +0200 Subject: [PATCH 34/46] Bump version to v4.1.17 (#30472) --- CHANGELOG.md | 33 +++++++++++++++++++++++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e65ebd8f9..b0254d467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,39 @@ Changelog All notable changes to this project will be documented in this file. +## [4.1.17] - 2024-05-30 + +### Security + +- Update dependencies +- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf)) +- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh)) +- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553)) + +### Added + +- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592)) +- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092)) + +### Removed + +- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862)) + +### Fixed + +- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450)) +- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403)) +- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306)) +- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125)) +- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119)) +- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084)) +- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022)) +- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838)) +- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597)) +- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530)) +- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379)) +- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363)) + ## [4.1.16] - 2024-02-23 ### Added diff --git a/docker-compose.yml b/docker-compose.yml index acaeafa63..6dde096d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.1.16 + image: ghcr.io/mastodon/mastodon:v4.1.17 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.1.16 + image: ghcr.io/mastodon/mastodon:v4.1.17 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.1.16 + image: ghcr.io/mastodon/mastodon:v4.1.17 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index d5babbe1d..3e2a11f14 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 16 + 17 end def flags From 2ee88a99d973e99a996b49973ed3bf992ea095e8 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 12 Oct 2023 16:47:18 +0200 Subject: [PATCH 35/46] Change PWA start URL from `/home` to `/` (#27377) --- app/serializers/manifest_serializer.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/serializers/manifest_serializer.rb b/app/serializers/manifest_serializer.rb index 48f3aa7a6..b6533815a 100644 --- a/app/serializers/manifest_serializer.rb +++ b/app/serializers/manifest_serializer.rb @@ -16,11 +16,18 @@ class ManifestSerializer < ActiveModel::Serializer 512 ).freeze - attributes :name, :short_name, + attributes :id, :name, :short_name, :icons, :theme_color, :background_color, :display, :start_url, :scope, :share_target, :shortcuts + def id + # This is set to `/home` because that was the old value of `start_url` and + # thus the fallback ID computed by Chrome: + # https://developer.chrome.com/blog/pwa-manifest-id/ + '/home' + end + def name object.title end @@ -53,7 +60,7 @@ class ManifestSerializer < ActiveModel::Serializer end def start_url - '/home' + '/' end def scope From 8e924e4338b41080282719bb0421a92cf0df448f Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 24 Jun 2024 15:11:10 +0200 Subject: [PATCH 36/46] Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments (#30819) --- app/models/status_edit.rb | 2 +- .../controllers/admin/statuses_controller_spec.rb | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index e33470226..83f06f0fd 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -41,7 +41,7 @@ class StatusEdit < ApplicationRecord default_scope { order(id: :asc) } delegate :local?, :application, :edited?, :edited_at, - :discarded?, :visibility, to: :status + :discarded?, :visibility, :language, to: :status def emojis return @emojis if defined?(@emojis) diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 877c7e63e..514c1d112 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -39,6 +39,21 @@ describe Admin::StatusesController do end end + describe 'GET #show' do + before do + status.media_attachments << Fabricate(:media_attachment, type: :image, account: status.account) + status.save! + status.snapshot!(at_time: status.created_at, rate_limit: false) + status.update!(text: 'Hello, this is an edited post') + status.snapshot!(rate_limit: false) + get :show, params: { account_id: account.id, id: status.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + end + describe 'POST #batch' do subject { post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } } } From 839147e09932fbc0a9e9f75b8a76005a5a4704f8 Mon Sep 17 00:00:00 2001 From: Tim Rogers Date: Mon, 24 Jun 2024 09:41:04 -0500 Subject: [PATCH 37/46] Added check for STATSD_ADDR setting to emit a warning and proceed rather than crashing if the address is unreachable (#30691) --- config/initializers/statsd.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb index 93ea1d1e4..d65d16aa5 100644 --- a/config/initializers/statsd.rb +++ b/config/initializers/statsd.rb @@ -3,13 +3,17 @@ if ENV['STATSD_ADDR'].present? host, port = ENV['STATSD_ADDR'].split(':') - $statsd = ::Statsd.new(host, port) - $statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } + begin + statsd = Statsd.new(host, port) + statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } - ::NSA.inform_statsd($statsd) do |informant| - informant.collect(:action_controller, :web) - informant.collect(:active_record, :db) - informant.collect(:active_support_cache, :cache) - informant.collect(:sidekiq, :sidekiq) + NSA.inform_statsd(statsd) do |informant| + informant.collect(:action_controller, :web) + informant.collect(:active_record, :db) + informant.collect(:active_support_cache, :cache) + informant.collect(:sidekiq, :sidekiq) + end + rescue + Rails.logger.warn("statsd address #{ENV['STATSD_ADDR']} not reachable, proceeding without statsd") end end From d8e8437a29d20cf6151ebfcf82d4a941377848fa Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 2 Jul 2024 15:34:44 +0200 Subject: [PATCH 38/46] Update dependency rails --- Gemfile.lock | 120 ++++++++++++++++++++++++++------------------------- 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6ac2df615..80d8f7f27 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,40 +10,40 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (6.1.7.7) - actionpack (= 6.1.7.7) - activesupport (= 6.1.7.7) + actioncable (6.1.7.8) + actionpack (= 6.1.7.8) + activesupport (= 6.1.7.8) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.7.7) - actionpack (= 6.1.7.7) - activejob (= 6.1.7.7) - activerecord (= 6.1.7.7) - activestorage (= 6.1.7.7) - activesupport (= 6.1.7.7) + actionmailbox (6.1.7.8) + actionpack (= 6.1.7.8) + activejob (= 6.1.7.8) + activerecord (= 6.1.7.8) + activestorage (= 6.1.7.8) + activesupport (= 6.1.7.8) mail (>= 2.7.1) - actionmailer (6.1.7.7) - actionpack (= 6.1.7.7) - actionview (= 6.1.7.7) - activejob (= 6.1.7.7) - activesupport (= 6.1.7.7) + actionmailer (6.1.7.8) + actionpack (= 6.1.7.8) + actionview (= 6.1.7.8) + activejob (= 6.1.7.8) + activesupport (= 6.1.7.8) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.7.7) - actionview (= 6.1.7.7) - activesupport (= 6.1.7.7) + actionpack (6.1.7.8) + actionview (= 6.1.7.8) + activesupport (= 6.1.7.8) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.7.7) - actionpack (= 6.1.7.7) - activerecord (= 6.1.7.7) - activestorage (= 6.1.7.7) - activesupport (= 6.1.7.7) + actiontext (6.1.7.8) + actionpack (= 6.1.7.8) + activerecord (= 6.1.7.8) + activestorage (= 6.1.7.8) + activesupport (= 6.1.7.8) nokogiri (>= 1.8.5) - actionview (6.1.7.7) - activesupport (= 6.1.7.7) + actionview (6.1.7.8) + activesupport (= 6.1.7.8) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -54,22 +54,22 @@ GEM case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_query_trace (1.8) - activejob (6.1.7.7) - activesupport (= 6.1.7.7) + activejob (6.1.7.8) + activesupport (= 6.1.7.8) globalid (>= 0.3.6) - activemodel (6.1.7.7) - activesupport (= 6.1.7.7) - activerecord (6.1.7.7) - activemodel (= 6.1.7.7) - activesupport (= 6.1.7.7) - activestorage (6.1.7.7) - actionpack (= 6.1.7.7) - activejob (= 6.1.7.7) - activerecord (= 6.1.7.7) - activesupport (= 6.1.7.7) + activemodel (6.1.7.8) + activesupport (= 6.1.7.8) + activerecord (6.1.7.8) + activemodel (= 6.1.7.8) + activesupport (= 6.1.7.8) + activestorage (6.1.7.8) + actionpack (= 6.1.7.8) + activejob (= 6.1.7.8) + activerecord (= 6.1.7.8) + activesupport (= 6.1.7.8) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.7.7) + activesupport (6.1.7.8) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -105,6 +105,7 @@ GEM aws-sigv4 (~> 1.4) aws-sigv4 (1.5.2) aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) bcrypt (3.1.17) better_errors (2.9.1) coderay (>= 1.0.0) @@ -395,7 +396,7 @@ GEM net-smtp makara (0.5.1) activerecord (>= 5.2.0) - marcel (1.0.2) + marcel (1.0.4) mario-redis-lock (1.2.1) redis (>= 3.0.5) matrix (0.4.2) @@ -405,7 +406,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) mini_mime (1.1.5) - mini_portile2 (2.8.5) + mini_portile2 (2.8.7) minitest (5.17.0) msgpack (1.6.0) multi_json (1.15.0) @@ -424,7 +425,7 @@ GEM net-protocol net-ssh (7.0.1) nio4r (2.5.9) - nokogiri (1.16.5) + nokogiri (1.16.6) mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.2.8) @@ -497,7 +498,7 @@ GEM activesupport (>= 3.0.0) raabro (1.4.0) racc (1.7.3) - rack (2.2.8.1) + rack (2.2.9) rack-attack (6.6.1) rack (>= 1.0, < 3) rack-cors (1.1.1) @@ -512,20 +513,20 @@ GEM rack rack-test (2.0.2) rack (>= 1.3) - rails (6.1.7.7) - actioncable (= 6.1.7.7) - actionmailbox (= 6.1.7.7) - actionmailer (= 6.1.7.7) - actionpack (= 6.1.7.7) - actiontext (= 6.1.7.7) - actionview (= 6.1.7.7) - activejob (= 6.1.7.7) - activemodel (= 6.1.7.7) - activerecord (= 6.1.7.7) - activestorage (= 6.1.7.7) - activesupport (= 6.1.7.7) + rails (6.1.7.8) + actioncable (= 6.1.7.8) + actionmailbox (= 6.1.7.8) + actionmailer (= 6.1.7.8) + actionpack (= 6.1.7.8) + actiontext (= 6.1.7.8) + actionview (= 6.1.7.8) + activejob (= 6.1.7.8) + activemodel (= 6.1.7.8) + activerecord (= 6.1.7.8) + activestorage (= 6.1.7.8) + activesupport (= 6.1.7.8) bundler (>= 1.15.0) - railties (= 6.1.7.7) + railties (= 6.1.7.8) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -541,9 +542,9 @@ GEM railties (>= 6.0.0, < 7) rails-settings-cached (0.6.6) rails (>= 4.2.0) - railties (6.1.7.7) - actionpack (= 6.1.7.7) - activesupport (= 6.1.7.7) + railties (6.1.7.8) + actionpack (= 6.1.7.8) + activesupport (= 6.1.7.8) method_source rake (>= 12.2) thor (~> 1.0) @@ -664,7 +665,8 @@ GEM simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) smart_properties (1.17.0) - sprockets (3.7.2) + sprockets (3.7.3) + base64 concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.4.2) @@ -755,7 +757,7 @@ GEM xorcist (1.1.3) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.13) + zeitwerk (2.6.16) PLATFORMS ruby From 815680bd1383840502982e98b0079d9a4dc10cec Mon Sep 17 00:00:00 2001 From: David Roetzel Date: Thu, 27 Jun 2024 16:40:19 +0200 Subject: [PATCH 39/46] Add size limit for link preview URLs (#30854) --- app/services/fetch_link_card_service.rb | 5 ++++- spec/services/fetch_link_card_service_spec.rb | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 4d55aa5e2..1222b2fab 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -15,6 +15,9 @@ class FetchLinkCardService < BaseService ) }iox + # URL size limit to safely store in PosgreSQL's unique indexes + BYTESIZE_LIMIT = 2692 + def call(status) @status = status @original_url = parse_urls @@ -85,7 +88,7 @@ class FetchLinkCardService < BaseService def bad_url?(uri) # Avoid local instance URLs and invalid URLs - uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) + uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT end def mention_link?(anchor) diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 7a758f910..d8f4b8e23 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -95,6 +95,19 @@ RSpec.describe FetchLinkCardService, type: :service do expect(a_request(:get, 'http://example.com/test?data=file.gpx%5E1')).to have_been_made.once end end + + context 'with an URL too long for PostgreSQL unique indexes' do + let(:url) { "http://example.com/#{'a' * 2674}" } + let(:status) { Fabricate(:status, text: url) } + + it 'does not fetch the URL' do + expect(a_request(:get, url)).to_not have_been_made + end + + it 'does not create a preview card' do + expect(status.preview_card).to be_nil + end + end end context 'in a remote status' do From 2a5819e8bb35a0f04cda7c1c45103a1bf2a3cc91 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 27 Jun 2024 23:34:34 +0200 Subject: [PATCH 40/46] Change search modifiers to be case-insensitive (#30865) --- app/lib/search_query_transformer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index aef05e9d9..714cab43c 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -101,7 +101,7 @@ class SearchQueryTransformer < Parslet::Transform end rule(clause: subtree(:clause)) do - prefix = clause[:prefix][:term].to_s if clause[:prefix] + prefix = clause[:prefix][:term].to_s.downcase if clause[:prefix] operator = clause[:operator]&.to_s if clause[:prefix] From 6cf83a2a64bf2f0f4848fb0fabae4d5f181ba587 Mon Sep 17 00:00:00 2001 From: David Roetzel Date: Fri, 21 Jun 2024 14:51:10 +0200 Subject: [PATCH 41/46] Improve encoding detection for link cards (#30780) --- app/lib/link_details_extractor.rb | 15 ++++++++++----- .../fixtures/requests/low_confidence_latin1.txt | 17 +++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 11 ++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 spec/fixtures/requests/low_confidence_latin1.txt diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index 9d7748425..de354b109 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -255,16 +255,21 @@ class LinkDetailsExtractor end def document - @document ||= Nokogiri::HTML(@html, nil, encoding) + @document ||= detect_encoding_and_parse_document end - def encoding - @encoding ||= begin - guess = detector.detect(@html, @html_charset) - guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil + def detect_encoding_and_parse_document + [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding| + document = Nokogiri::HTML(@html, nil, encoding) + return document if document.to_s.valid_encoding? end end + def detect_encoding + guess = detector.detect(@html, @html_charset) + guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil + end + def detector @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector| detector.strip_tags = true diff --git a/spec/fixtures/requests/low_confidence_latin1.txt b/spec/fixtures/requests/low_confidence_latin1.txt new file mode 100644 index 000000000..39c3e23d6 --- /dev/null +++ b/spec/fixtures/requests/low_confidence_latin1.txt @@ -0,0 +1,17 @@ +HTTP/1.1 200 OK +server: nginx +date: Thu, 13 Jun 2024 14:33:13 GMT +content-type: text/html; charset=ISO-8859-1 +content-length: 158 +accept-ranges: bytes + + + + + + Tofu á l'orange + + +

Tofu á l'orange

+ + diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index d8f4b8e23..c50eff90e 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -13,6 +13,7 @@ RSpec.describe FetchLinkCardService, type: :service do stub_request(:get, 'http://example.com/test?data=file.gpx%5E1').to_return(status: 200) stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt')) stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) + stub_request(:get, 'http://example.com/low_confidence_latin1').to_return(request_fixture('low_confidence_latin1.txt')) subject.call(status) end @@ -62,7 +63,15 @@ RSpec.describe FetchLinkCardService, type: :service do end end - context do + context 'with a URL of a page in ISO-8859-1 encoding, that charlock_holmes cannot detect' do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') } + + it 'decodes the HTML' do + expect(status.preview_card.title).to eq("Tofu á l'orange") + end + end + + context 'with a Japanese path URL' do let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } it 'works with Japanese path string' do From 4b45333aff37671e26ec2650e8a0b10ee02d4509 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 3 Jul 2024 09:15:47 +0200 Subject: [PATCH 42/46] fix: Return HTTP 422 when scheduled status time is less than 5 minutes (#30584) --- app/services/post_status_service.rb | 2 +- .../api/v1/statuses_controller_spec.rb | 40 +++++++++++++++++++ spec/services/post_status_service_spec.rb | 10 +++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 7b57c05f5..bebdeb4d0 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -148,7 +148,7 @@ class PostStatusService < BaseService end def scheduled_in_the_past? - @scheduled_at.present? && @scheduled_at <= Time.now.utc + MIN_SCHEDULE_OFFSET + @scheduled_at.present? && @scheduled_at <= Time.now.utc end def bump_potential_friendship! diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb index 24810a5d2..d9fefce69 100644 --- a/spec/controllers/api/v1/statuses_controller_spec.rb +++ b/spec/controllers/api/v1/statuses_controller_spec.rb @@ -163,6 +163,46 @@ RSpec.describe Api::V1::StatusesController, type: :controller do expect(response.headers['X-RateLimit-Remaining']).to eq '0' end end + + context 'with missing thread' do + subject { post :create, params: params } + + let(:params) { { status: 'Hello world', in_reply_to_id: 0 } } + + it 'returns http not found' do + subject + + expect(response).to have_http_status(404) + end + end + + context 'when scheduling a status' do + subject { post :create, params: params } + + let(:params) { { status: 'Hello world', scheduled_at: 10.minutes.from_now } } + let(:account) { user.account } + + it 'returns HTTP 200' do + subject + + expect(response).to have_http_status(200) + end + + it 'creates a scheduled status' do + expect { subject }.to change { account.scheduled_statuses.count }.from(0).to(1) + end + + context 'when the scheduling time is less than 5 minutes' do + let(:params) { { status: 'Hello world', scheduled_at: 4.minutes.from_now } } + + it 'does not create a scheduled status', :aggregate_failures do + subject + + expect(response).to have_http_status(422) + expect(account.scheduled_statuses).to be_empty + end + end + end end describe 'DELETE #destroy' do diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index c56f9efdb..6f0b4677f 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -59,6 +59,16 @@ RSpec.describe PostStatusService, type: :service do status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future) expect(status2.id).to eq status1.id end + + context 'when scheduled_at is less than min offset' do + let(:invalid_scheduled_time) { 4.minutes.from_now } + + it 'raises invalid record error' do + expect do + subject.call(account, text: 'Hi future!', scheduled_at: invalid_scheduled_time) + end.to raise_error(ActiveRecord::RecordInvalid) + end + end end it 'creates response to the original status of boost' do From 122740047a1a0361e32991e791322e0e6e41ac9f Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:11:28 +0200 Subject: [PATCH 43/46] Merge pull request from GHSA-vp5r-5pgw-jwqx * Fix streaming sessions not being closed when revoking access to an app * Add tests for GHSA-7w3c-p9j8-mq3x --- .../oauth/authorized_applications_controller.rb | 1 + app/lib/application_extension.rb | 8 +++++--- .../oauth/authorized_applications_controller_spec.rb | 10 ++++++++++ .../settings/applications_controller_spec.rb | 8 ++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb index 63afc4c06..28b734b1b 100644 --- a/app/controllers/oauth/authorized_applications_controller.rb +++ b/app/controllers/oauth/authorized_applications_controller.rb @@ -16,6 +16,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio def destroy Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner) + Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner) super end diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index d1222656b..c7bd347cd 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -14,17 +14,19 @@ module ApplicationExtension # dependent: delete_all, which means the ActiveRecord callback in # AccessTokenExtension is not run, so instead we manually announce to # streaming that these tokens are being deleted. - before_destroy :push_to_streaming_api, prepend: true + before_destroy :close_streaming_sessions, prepend: true end def confirmation_redirect_uri redirect_uri.lines.first.strip end - def push_to_streaming_api + def close_streaming_sessions(resource_owner = nil) # TODO: #28793 Combine into a single topic payload = Oj.dump(event: :kill) - access_tokens.in_batches do |tokens| + scope = access_tokens + scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil? + scope.in_batches do |tokens| redis.pipelined do |pipeline| tokens.ids.each do |id| pipeline.publish("timeline:access_token:#{id}", payload) diff --git a/spec/controllers/oauth/authorized_applications_controller_spec.rb b/spec/controllers/oauth/authorized_applications_controller_spec.rb index 901e538e9..7689edb01 100644 --- a/spec/controllers/oauth/authorized_applications_controller_spec.rb +++ b/spec/controllers/oauth/authorized_applications_controller_spec.rb @@ -45,9 +45,11 @@ describe Oauth::AuthorizedApplicationsController do let!(:application) { Fabricate(:application) } let!(:access_token) { Fabricate(:accessible_access_token, application: application, resource_owner_id: user.id) } let!(:web_push_subscription) { Fabricate(:web_push_subscription, user: user, access_token: access_token) } + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } before do sign_in user, scope: :user + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) post :destroy, params: { id: application.id } end @@ -58,5 +60,13 @@ describe Oauth::AuthorizedApplicationsController do it 'removes subscriptions for the application\'s access tokens' do expect(Web::PushSubscription.where(user: user).count).to eq 0 end + + it 'removes the web_push_subscription' do + expect { web_push_subscription.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'sends a session kill payload to the streaming server' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}') + end end end diff --git a/spec/controllers/settings/applications_controller_spec.rb b/spec/controllers/settings/applications_controller_spec.rb index 1292e9ff8..e0019035d 100644 --- a/spec/controllers/settings/applications_controller_spec.rb +++ b/spec/controllers/settings/applications_controller_spec.rb @@ -160,7 +160,11 @@ describe Settings::ApplicationsController do end describe 'destroy' do + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } + let!(:access_token) { Fabricate(:accessible_access_token, application: app) } + before do + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) post :destroy, params: { id: app.id } end @@ -171,6 +175,10 @@ describe Settings::ApplicationsController do it 'removes the app' do expect(Doorkeeper::Application.find_by(id: app.id)).to be_nil end + + it 'sends a session kill payload to the streaming server' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}') + end end describe 'regenerate' do From 34aeef345320f7808a1eec00c8b5025e9a9b8329 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:26:49 +0200 Subject: [PATCH 44/46] Merge pull request from GHSA-58x8-3qxw-6hm7 * Fix insufficient permission checking for public timeline endpoints Note that this changes unauthenticated access failure code from 401 to 422 * Add more tests for public timelines * Require user token in `/api/v1/statuses/:id/translate` and `/api/v1/scheduled_statuses` --- .../api/v1/scheduled_statuses_controller.rb | 1 + .../v1/statuses/translations_controller.rb | 1 + .../api/v1/timelines/public_controller.rb | 1 + .../api/v1/timelines/tag_controller.rb | 3 +- .../v1/scheduled_statuses_controller_spec.rb | 34 +++++ .../statuses/translations_controller_spec.rb | 53 +++++++ .../v1/timelines/public_controller_spec.rb | 4 +- .../api/v1/timelines/tag_controller_spec.rb | 19 ++- spec/requests/api/v1/timelines/public_spec.rb | 133 ++++++++++++++++++ 9 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 spec/controllers/api/v1/scheduled_statuses_controller_spec.rb create mode 100644 spec/controllers/api/v1/statuses/translations_controller_spec.rb create mode 100644 spec/requests/api/v1/timelines/public_spec.rb diff --git a/app/controllers/api/v1/scheduled_statuses_controller.rb b/app/controllers/api/v1/scheduled_statuses_controller.rb index f90642a73..9ccbf96ee 100644 --- a/app/controllers/api/v1/scheduled_statuses_controller.rb +++ b/app/controllers/api/v1/scheduled_statuses_controller.rb @@ -6,6 +6,7 @@ class Api::V1::ScheduledStatusesController < Api::BaseController before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, except: [:update, :destroy] before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:update, :destroy] + before_action :require_user! before_action :set_statuses, only: :index before_action :set_status, except: :index diff --git a/app/controllers/api/v1/statuses/translations_controller.rb b/app/controllers/api/v1/statuses/translations_controller.rb index 540b17d00..f6b816b95 100644 --- a/app/controllers/api/v1/statuses/translations_controller.rb +++ b/app/controllers/api/v1/statuses/translations_controller.rb @@ -4,6 +4,7 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController include Authorization before_action -> { doorkeeper_authorize! :read, :'read:statuses' } + before_action :require_user! before_action :set_status before_action :set_translation diff --git a/app/controllers/api/v1/timelines/public_controller.rb b/app/controllers/api/v1/timelines/public_controller.rb index d253b744f..bb356f8a6 100644 --- a/app/controllers/api/v1/timelines/public_controller.rb +++ b/app/controllers/api/v1/timelines/public_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Api::V1::Timelines::PublicController < Api::BaseController + before_action -> { authorize_if_got_token! :read, :'read:statuses' } before_action :require_user!, only: [:show], if: :require_auth? after_action :insert_pagination_headers, unless: -> { @statuses.empty? } diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb index 3f41eb688..c5bf4dd42 100644 --- a/app/controllers/api/v1/timelines/tag_controller.rb +++ b/app/controllers/api/v1/timelines/tag_controller.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true class Api::V1::Timelines::TagController < Api::BaseController - before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth? + before_action -> { authorize_if_got_token! :read, :'read:statuses' } + before_action :require_user!, if: :require_auth? before_action :load_tag after_action :insert_pagination_headers, unless: -> { @statuses.empty? } diff --git a/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb b/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb new file mode 100644 index 000000000..cc3b65f37 --- /dev/null +++ b/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::ScheduledStatusesController do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } + let(:account) { Fabricate(:account) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + context 'with an application token' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses') } + + it 'returns http unprocessable entity' do + get :index + + expect(response) + .to have_http_status(422) + end + end + + describe 'GET #index' do + it 'returns http success' do + get :index + + expect(response).to have_http_status(200) + end + end +end diff --git a/spec/controllers/api/v1/statuses/translations_controller_spec.rb b/spec/controllers/api/v1/statuses/translations_controller_spec.rb new file mode 100644 index 000000000..53a14ffc9 --- /dev/null +++ b/spec/controllers/api/v1/statuses/translations_controller_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::Statuses::TranslationsController do + render_views + + let(:user) { Fabricate(:user) } + let(:app) { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses', application: app) } + + context 'with an application token' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses', application: app) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'POST /api/v1/statuses/:status_id/translate' do + let(:status) { Fabricate(:status, account: user.account, text: 'Hola', language: 'es') } + + before do + post :create, params: { status_id: status.id } + end + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end + end + end + + context 'with an oauth token' do + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'POST #create' do + let(:status) { Fabricate(:status, account: user.account, text: 'Hola', language: 'es') } + + before do + translation = TranslationService::Translation.new(text: 'Hello') + service = instance_double(TranslationService::DeepL, translate: translation) + allow(TranslationService).to receive_messages(configured?: true, configured: service) + Rails.cache.write('translation_service/languages', { 'es' => ['en'] }) + post :create, params: { status_id: status.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + end + end +end diff --git a/spec/controllers/api/v1/timelines/public_controller_spec.rb b/spec/controllers/api/v1/timelines/public_controller_spec.rb index 31e594d22..b488d4e25 100644 --- a/spec/controllers/api/v1/timelines/public_controller_spec.rb +++ b/spec/controllers/api/v1/timelines/public_controller_spec.rb @@ -12,7 +12,7 @@ describe Api::V1::Timelines::PublicController do end context 'with a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } describe 'GET #show' do before do @@ -42,7 +42,7 @@ describe Api::V1::Timelines::PublicController do end context 'without a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses') } describe 'GET #show' do it 'returns http success' do diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb index 1c60798fc..89622a41a 100644 --- a/spec/controllers/api/v1/timelines/tag_controller_spec.rb +++ b/spec/controllers/api/v1/timelines/tag_controller_spec.rb @@ -6,7 +6,8 @@ describe Api::V1::Timelines::TagController do render_views let(:user) { Fabricate(:user) } - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } + let(:scopes) { 'read:statuses' } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } before do allow(controller).to receive(:doorkeeper_token) { token } @@ -48,13 +49,23 @@ describe Api::V1::Timelines::TagController do Form::AdminSettings.new(timeline_preview: false).save end - context 'when the user is not authenticated' do + context 'without an access token' do let(:token) { nil } - it 'returns http unauthorized' do + it 'returns http unprocessable entity' do subject - expect(response).to have_http_status(401) + expect(response).to have_http_status(422) + end + end + + context 'with an application access token, not bound to a user' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: scopes) } + + it 'returns http unprocessable entity' do + subject + + expect(response).to have_http_status(422) end end diff --git a/spec/requests/api/v1/timelines/public_spec.rb b/spec/requests/api/v1/timelines/public_spec.rb new file mode 100644 index 000000000..debc6df06 --- /dev/null +++ b/spec/requests/api/v1/timelines/public_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe 'Public' do + let(:user) { Fabricate(:user) } + let(:scopes) { 'read:statuses' } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + let(:headers) { { 'Authorization' => "Bearer #{token.token}" } } + + shared_examples 'forbidden for wrong scope' do |wrong_scope| + let(:scopes) { wrong_scope } + + it 'returns http forbidden' do + subject + + expect(response).to have_http_status(403) + end + end + + shared_examples 'a successful request to the public timeline' do + it 'returns the expected statuses successfully', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(body_as_json.pluck(:id)).to match_array(expected_statuses.map { |status| status.id.to_s }) + end + end + + describe 'GET /api/v1/timelines/public' do + subject do + get '/api/v1/timelines/public', headers: headers, params: params + end + + let!(:private_status) { Fabricate(:status, visibility: :private) } # rubocop:disable RSpec/LetSetup + let!(:local_status) { Fabricate(:status, account: Fabricate.build(:account, domain: nil)) } + let!(:remote_status) { Fabricate(:status, account: Fabricate.build(:account, domain: 'example.com')) } + let!(:media_status) { Fabricate(:status, media_attachments: [Fabricate.build(:media_attachment)]) } + + let(:params) { {} } + + context 'when the instance allows public preview' do + let(:expected_statuses) { [local_status, remote_status, media_status] } + + it_behaves_like 'forbidden for wrong scope', 'profile' + + context 'with an authorized user' do + it_behaves_like 'a successful request to the public timeline' + end + + context 'with an anonymous user' do + let(:headers) { {} } + + it_behaves_like 'a successful request to the public timeline' + end + + context 'with local param' do + let(:params) { { local: true } } + let(:expected_statuses) { [local_status, media_status] } + + it_behaves_like 'a successful request to the public timeline' + end + + context 'with remote param' do + let(:params) { { remote: true } } + let(:expected_statuses) { [remote_status] } + + it_behaves_like 'a successful request to the public timeline' + end + + context 'with only_media param' do + let(:params) { { only_media: true } } + let(:expected_statuses) { [media_status] } + + it_behaves_like 'a successful request to the public timeline' + end + + context 'with limit param' do + let(:params) { { limit: 1 } } + + it 'returns only the requested number of statuses', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(body_as_json.size).to eq(params[:limit]) + end + + it 'sets the correct pagination headers', :aggregate_failures do + subject + + headers = response.headers['Link'] + + expect(headers.find_link(%w(rel prev)).href).to eq(api_v1_timelines_public_url(limit: 1, min_id: media_status.id.to_s)) + expect(headers.find_link(%w(rel next)).href).to eq(api_v1_timelines_public_url(limit: 1, max_id: media_status.id.to_s)) + end + end + end + + context 'when the instance does not allow public preview' do + before do + Form::AdminSettings.new(timeline_preview: false).save + end + + it_behaves_like 'forbidden for wrong scope', 'profile' + + context 'without an authentication token' do + let(:headers) { {} } + + it 'returns http unprocessable entity' do + subject + + expect(response).to have_http_status(422) + end + end + + context 'with an application access token, not bound to a user' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: scopes) } + + it 'returns http unprocessable entity' do + subject + + expect(response).to have_http_status(422) + end + end + + context 'with an authenticated user' do + let(:expected_statuses) { [local_status, remote_status, media_status] } + + it_behaves_like 'a successful request to the public timeline' + end + end + end +end From a1c7aae28aecf06659c5b18cfa131b37cd1512a3 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:45:52 +0200 Subject: [PATCH 45/46] Merge pull request from GHSA-xjvf-fm67-4qc3 --- app/lib/activitypub/activity/create.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 487b65223..373232b47 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -102,7 +102,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def find_existing_status status = status_from_uri(object_uri) status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present? - status + status if status&.account_id == @account.id end def process_status_params From ff90ebffaa80bb48d12a838d9a4f82fa7edb602b Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:46:39 +0200 Subject: [PATCH 46/46] Bump version to v4.1.18 (#30911) --- CHANGELOG.md | 22 ++++++++++++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0254d467..7306bcd8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,28 @@ Changelog All notable changes to this project will be documented in this file. +## [4.1.18] - 2024-07-04 + +### Security + +- Fix incorrect permission checking on multiple API endpoints ([GHSA-58x8-3qxw-6hm7](https://github.com/mastodon/mastodon/security/advisories/GHSA-58x8-3qxw-6hm7)) +- Fix incorrect authorship checking when processing some activities (CVE-2024-37903, [GHSA-xjvf-fm67-4qc3](https://github.com/mastodon/mastodon/security/advisories/GHSA-xjvf-fm67-4qc3)) +- Fix ongoing streaming sessions not being invalidated when application tokens get revoked ([GHSA-vp5r-5pgw-jwqx](https://github.com/mastodon/mastodon/security/advisories/GHSA-vp5r-5pgw-jwqx)) +- Update dependencies + +### Changed + +- Change preview cards generation to skip unusually long URLs ([oneiros](https://github.com/mastodon/mastodon/pull/30854)) +- Change search modifiers to be case-insensitive ([Gargron](https://github.com/mastodon/mastodon/pull/30865)) +- Change `STATSD_ADDR` handling to emit a warning rather than crashing if the address is unreachable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30691)) +- Change PWA start URL from `/home` to `/` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27377)) + +### Fixed + +- Fix scheduled statuses scheduled in less than 5 minutes being immediately published ([danielmbrasil](https://github.com/mastodon/mastodon/pull/30584)) +- Fix encoding detection for link cards ([oneiros](https://github.com/mastodon/mastodon/pull/30780)) +- Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30819)) + ## [4.1.17] - 2024-05-30 ### Security diff --git a/docker-compose.yml b/docker-compose.yml index 6dde096d2..1d4bb1b1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.1.17 + image: ghcr.io/mastodon/mastodon:v4.1.18 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.1.17 + image: ghcr.io/mastodon/mastodon:v4.1.18 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.1.17 + image: ghcr.io/mastodon/mastodon:v4.1.18 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 3e2a11f14..627f3dc97 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 17 + 18 end def flags