Merge pull request #137 from Y-zu-don-maintenance-org/4.1.18

4.1.18

.circleci/config.yml

@@ -1,225 +0,0 @@
-version: 2.1
-
-orbs:
-  ruby: circleci/ruby@2.0.0
-  node: circleci/node@5.0.3
-
-executors:
-  default:
-    parameters:
-      ruby-version:
-        type: string
-    docker:
-      - image: cimg/ruby:<< parameters.ruby-version >>
-        environment:
-          BUNDLE_JOBS: 3
-          BUNDLE_RETRY: 3
-          CONTINUOUS_INTEGRATION: true
-          DB_HOST: localhost
-          DB_USER: root
-          DISABLE_SIMPLECOV: true
-          RAILS_ENV: test
-      - image: cimg/postgres:14.5
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_HOST_AUTH_METHOD: trust
-      - image: cimg/redis:7.0
-
-commands:
-  install-system-dependencies:
-    steps:
-      - run:
-          name: Install system dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y libicu-dev libidn11-dev
-  install-ruby-dependencies:
-    parameters:
-      ruby-version:
-        type: string
-    steps:
-      - run:
-          command: |
-            bundle config clean 'true'
-            bundle config frozen 'true'
-            bundle config without 'development production'
-          name: Set bundler settings
-      - ruby/install-deps:
-          bundler-version: '2.3.26'
-          key: ruby<< parameters.ruby-version >>-gems-v1
-  wait-db:
-    steps:
-      - run:
-          command: dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m
-          name: Wait for PostgreSQL and Redis
-
-jobs:
-  build:
-    docker:
-      - image: cimg/ruby:3.0-node
-        environment:
-          RAILS_ENV: test
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - node/install-packages:
-          cache-version: v1
-          pkg-manager: yarn
-      - run:
-          command: |
-            export NODE_OPTIONS=--openssl-legacy-provider
-            ./bin/rails assets:precompile
-          name: Precompile assets
-      - persist_to_workspace:
-          paths:
-            - public/assets
-            - public/packs-test
-          root: .
-
-  test:
-    parameters:
-      ruby-version:
-        type: string
-    executor:
-      name: default
-      ruby-version: << parameters.ruby-version >>
-    environment:
-      ALLOW_NOPAM: true
-      PAM_ENABLED: true
-      PAM_DEFAULT_SERVICE: pam_test
-      PAM_CONTROLLED_SERVICE: pam_test_controlled
-    parallelism: 4
-    steps:
-      - checkout
-      - install-system-dependencies
-      - run:
-          command: sudo apt-get install -y ffmpeg imagemagick libpam-dev
-          name: Install additional system dependencies
-      - run:
-          command: bundle config with 'pam_authentication'
-          name: Enable PAM authentication
-      - install-ruby-dependencies:
-          ruby-version: << parameters.ruby-version >>
-      - attach_workspace:
-          at: .
-      - wait-db
-      - run:
-          command: ./bin/rails db:create db:schema:load db:seed
-          name: Load database schema
-      - ruby/rspec-test
-
-  test-migrations:
-    executor:
-      name: default
-      ruby-version: '3.0'
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - wait-db
-      - run:
-          command: ./bin/rails db:create
-          name: Create database
-      - run:
-          command: ./bin/rails db:migrate VERSION=20171010025614
-          name: Run migrations up to v2.0.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180514140000
-          name: Run migrations up to v2.4.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180707154237
-          name: Run migrations up to v2.4.3
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4_3
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all remaining migrations
-      - run:
-          command: ./bin/rails tests:migrations:check_database
-          name: Check migration result
-
-  test-two-step-migrations:
-    executor:
-      name: default
-      ruby-version: '3.0'
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - wait-db
-      - run:
-          command: ./bin/rails db:create
-          name: Create database
-      - run:
-          command: ./bin/rails db:migrate VERSION=20171010025614
-          name: Run migrations up to v2.0.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180514140000
-          name: Run pre-deployment migrations up to v2.4.0
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180707154237
-          name: Run migrations up to v2.4.3
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4_3
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all remaining pre-deployment migrations
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all post-deployment migrations
-      - run:
-          command: ./bin/rails tests:migrations:check_database
-          name: Check migration result
-
-workflows:
-  version: 2
-  build-and-test:
-    jobs:
-      - build
-      - test:
-          matrix:
-            parameters:
-              ruby-version:
-                - '2.7'
-                - '3.0'
-          name: test-ruby<< matrix.ruby-version >>
-          requires:
-            - build
-      - test-migrations:
-          requires:
-            - build
-      - test-two-step-migrations:
-          requires:
-            - build
-      - node/run:
-          cache-version: v1
-          name: test-webui
-          pkg-manager: yarn
-          requires:
-            - build
-          version: '16.18'
-          yarn-run: test:jest

.github/workflows/build-container-image.yml

@@ -0,0 +1,92 @@
+on:
+  workflow_call:
+    inputs:
+      platforms:
+        required: true
+        type: string
+      cache:
+        type: boolean
+        default: true
+      use_native_arm64_builder:
+        type: boolean
+      push_to_images:
+        type: string
+      flavor:
+        type: string
+      tags:
+        type: string
+      labels:
+        type: string
+
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: docker/setup-qemu-action@v2
+        if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder
+
+      - uses: docker/setup-buildx-action@v2
+        id: buildx
+        if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }}
+
+      - name: Start a local Docker Builder
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        run: |
+          docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234
+
+      - uses: docker/setup-buildx-action@v2
+        id: buildx-native
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        with:
+          driver: remote
+          endpoint: tcp://localhost:1234
+          platforms: linux/amd64
+          append: |
+            - endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865
+              platforms: linux/arm64
+              name: mastodon-docker-builder-arm64-01
+              driver-opts:
+                - servername=mastodon-docker-builder-arm64-01
+        env:
+          BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }}
+          BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }}
+          BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }}
+
+      - name: Log in to Docker Hub
+        if: contains(inputs.push_to_images, 'tootsuite')
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Log in to the Github Container registry
+        if: contains(inputs.push_to_images, 'ghcr.io')
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/metadata-action@v4
+        id: meta
+        if: ${{ inputs.push_to_images != '' }}
+        with:
+          images: ${{ inputs.push_to_images }}
+          flavor: ${{ inputs.flavor }}
+          tags: ${{ inputs.tags }}
+          labels: ${{ inputs.labels }}
+
+      - uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: ${{ inputs.platforms }}
+          provenance: false
+          builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}
+          push: ${{ inputs.push_to_images != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: ${{ inputs.cache && 'type=gha' || '' }}
+          cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }}

.github/workflows/build-image.yml

@@ -1,70 +0,0 @@
-name: Build container image
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'main'
-    tags:
-      - '*'
-  pull_request:
-    paths:
-      - .github/workflows/build-image.yml
-      - Dockerfile
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-  build-image:
-    runs-on: ubuntu-latest
-
-    concurrency:
-      group: ${{ github.ref }}
-      cancel-in-progress: true
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: hadolint/hadolint-action@v3.1.0
-      - uses: docker/setup-qemu-action@v2
-      - uses: docker/setup-buildx-action@v2
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
-
-      - name: Log in to the Github Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
-
-      - uses: docker/metadata-action@v4
-        id: meta
-        with:
-          images: |
-            tootsuite/mastodon
-            ghcr.io/mastodon/mastodon
-          flavor: |
-            latest=auto
-          tags: |
-            type=edge,branch=main
-            type=pep440,pattern={{raw}}
-            type=pep440,pattern=v{{major}}.{{minor}}
-            type=ref,event=pr
-
-      - uses: docker/build-push-action@v4
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          provenance: false
-          builder: ${{ steps.buildx.outputs.name }}
-          push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/build-releases.yml

@@ -0,0 +1,27 @@
+name: Build container release images
+on:
+  push:
+    tags:
+      - '*'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-image:
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: true
+      push_to_images: |
+        tootsuite/mastodon
+        ghcr.io/mastodon/mastodon
+      # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
+      cache: false
+      flavor: |
+        latest=false
+      tags: |
+        type=pep440,pattern={{raw}}
+        type=pep440,pattern=v{{major}}.{{minor}}
+    secrets: inherit

.github/workflows/lint-ruby.yml

@@ -1,41 +0,0 @@
-name: Ruby Linting
-on:
-  push:
-    branches-ignore:
-      - 'dependabot/**'
-    paths:
-      - 'Gemfile*'
-      - '.rubocop.yml'
-      - '**/*.rb'
-      - '**/*.rake'
-      - '.github/workflows/lint-ruby.yml'
-
-  pull_request:
-    paths:
-      - 'Gemfile*'
-      - '.rubocop.yml'
-      - '**/*.rb'
-      - '**/*.rake'
-      - '.github/workflows/lint-ruby.yml'
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set-up RuboCop Problem Mathcher
-        uses: r7kamura/rubocop-problem-matchers-action@v1
-
-      - name: Run rubocop
-        uses: github/super-linter@v4
-        env:
-          DEFAULT_BRANCH: main
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          LINTER_RULES_PATH: .
-          RUBY_CONFIG_FILE: .rubocop.yml
-          VALIDATE_ALL_CODEBASE: false
-          VALIDATE_RUBY: true

.github/workflows/test-image-build.yml

@@ -0,0 +1,15 @@
+name: Test container image build
+on:
+  pull_request:
+permissions:
+  contents: read
+
+jobs:
+  build-image:
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      platforms: linux/amd64 # Testing only on native platform so it is performant

.github/workflows/test-ruby.yml

@@ -0,0 +1,153 @@
+name: Ruby Testing
+
+on:
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+      - 'renovate/**'
+  pull_request:
+
+env:
+  BUNDLE_CLEAN: true
+  BUNDLE_FROZEN: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        mode:
+          - production
+          - test
+    env:
+      RAILS_ENV: ${{ matrix.mode }}
+      BUNDLE_WITH: ${{ matrix.mode }}
+      OTP_SECRET: precompile_placeholder
+      SECRET_KEY_BASE: precompile_placeholder
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+          node-version-file: '.nvmrc'
+
+      - name: Install native Ruby dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: .ruby-version
+          bundler-cache: true
+
+      - run: yarn --frozen-lockfile --production
+      - name: Precompile assets
+        # Previously had set this, but it's not supported
+        # export NODE_OPTIONS=--openssl-legacy-provider
+        run: |-
+          ./bin/rails assets:precompile
+
+      - uses: actions/upload-artifact@v3
+        if: matrix.mode == 'test'
+        with:
+          path: |-
+            ./public/assets
+            ./public/packs-test
+          name: ${{ github.sha }}
+          retention-days: 0
+
+  test:
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      ALLOW_NOPAM: true
+      PAM_ENABLED: true
+      PAM_DEFAULT_SERVICE: pam_test
+      PAM_CONTROLLED_SERVICE: pam_test_controlled
+      OIDC_ENABLED: true
+      OIDC_SCOPE: read
+      SAML_ENABLED: true
+      CAS_ENABLED: true
+      BUNDLE_WITH: 'pam_authentication test'
+      CI_JOBS: ${{ matrix.ci_job }}/4
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '.ruby-version'
+        ci_job:
+          - 1
+          - 2
+          - 3
+          - 4
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Update package index
+        run: sudo apt-get update
+
+      - name: Install native Ruby dependencies
+        run: sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Install additional system dependencies
+        run: sudo apt-get install -y ffmpeg imagemagick libpam-dev
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          bundler-cache: true
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bin/rspec

CHANGELOG.md

@@ -3,6 +3,229 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.18] - 2024-07-04
+
+### Security
+
+- Fix incorrect permission checking on multiple API endpoints ([GHSA-58x8-3qxw-6hm7](https://github.com/mastodon/mastodon/security/advisories/GHSA-58x8-3qxw-6hm7))
+- Fix incorrect authorship checking when processing some activities (CVE-2024-37903, [GHSA-xjvf-fm67-4qc3](https://github.com/mastodon/mastodon/security/advisories/GHSA-xjvf-fm67-4qc3))
+- Fix ongoing streaming sessions not being invalidated when application tokens get revoked ([GHSA-vp5r-5pgw-jwqx](https://github.com/mastodon/mastodon/security/advisories/GHSA-vp5r-5pgw-jwqx))
+- Update dependencies
+
+### Changed
+
+- Change preview cards generation to skip unusually long URLs ([oneiros](https://github.com/mastodon/mastodon/pull/30854))
+- Change search modifiers to be case-insensitive ([Gargron](https://github.com/mastodon/mastodon/pull/30865))
+- Change `STATSD_ADDR` handling to emit a warning rather than crashing if the address is unreachable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30691))
+- Change PWA start URL from `/home` to `/` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27377))
+
+### Fixed
+
+- Fix scheduled statuses scheduled in less than 5 minutes being immediately published ([danielmbrasil](https://github.com/mastodon/mastodon/pull/30584))
+- Fix encoding detection for link cards ([oneiros](https://github.com/mastodon/mastodon/pull/30780))
+- Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30819))
+
+## [4.1.17] - 2024-05-30
+
+### Security
+
+- Update dependencies
+- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf))
+- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh))
+- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553))
+
+### Added
+
+- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592))
+- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092))
+
+### Removed
+
+- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862))
+
+### Fixed
+
+- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450))
+- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403))
+- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306))
+- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125))
+- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119))
+- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084))
+- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022))
+- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838))
+- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597))
+- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530))
+- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379))
+- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363))
+
+## [4.1.16] - 2024-02-23
+
+### Added
+
+- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355))
+  In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week.
+  When this happens, users with the permission to change server settings will receive an email notification.
+  This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`.
+
+### Changed
+
+- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280))
+  If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations.
+  Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again.
+
+### Fixed
+
+- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335))
+- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358))
+
+## [4.1.15] - 2024-02-16
+
+### Fixed
+
+- Fix OmniAuth tests and edge cases in error handling ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29201), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29207))
+
+### Security
+
+- Fix insufficient checking of remote posts ([GHSA-jhrq-qvrm-qr36](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36))
+
+## [4.1.14] - 2024-02-14
+
+### Security
+
+- Update the `sidekiq-unique-jobs` dependency (see [GHSA-cmh9-rx85-xj38](https://github.com/mhenrixon/sidekiq-unique-jobs/security/advisories/GHSA-cmh9-rx85-xj38))
+  In addition, we have disabled the web interface for `sidekiq-unique-jobs` out of caution.
+  If you need it, you can re-enable it by setting `ENABLE_SIDEKIQ_UNIQUE_JOBS_UI=true`.
+  If you only need to clear all locks, you can now use `bundle exec rake sidekiq_unique_jobs:delete_all_locks`.
+- Update the `nokogiri` dependency (see [GHSA-xc9x-jj77-9p9j](https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j))
+- Disable administrative Doorkeeper routes ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29187))
+- Fix ongoing streaming sessions not being invalidated when applications get deleted in some cases ([GHSA-7w3c-p9j8-mq3x](https://github.com/mastodon/mastodon/security/advisories/GHSA-7w3c-p9j8-mq3x))
+  In some rare cases, the streaming server was not notified of access tokens revocation on application deletion.
+- Change external authentication behavior to never reattach a new identity to an existing user by default ([GHSA-vm39-j3vx-pch3](https://github.com/mastodon/mastodon/security/advisories/GHSA-vm39-j3vx-pch3))
+  Up until now, Mastodon has allowed new identities from external authentication providers to attach to an existing local user based on their verified e-mail address.
+  This allowed upgrading users from a database-stored password to an external authentication provider, or move from one authentication provider to another.
+  However, this behavior may be unexpected, and means that when multiple authentication providers are configured, the overall security would be that of the least secure authentication provider.
+  For these reasons, this behavior is now locked under the `ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH` environment variable.
+  In addition, regardless of this environment variable, Mastodon will refuse to attach two identities from the same authentication provider to the same account.
+
+## [4.1.13] - 2024-02-01
+
+### Security
+
+- Fix insufficient origin validation (CVE-2024-23832, [GHSA-3fjr-858r-92rw](https://github.com/mastodon/mastodon/security/advisories/GHSA-3fjr-858r-92rw))
+
+## [4.1.12] - 2024-01-24
+
+### Fixed
+
+- Fix error when processing remote files with unusually long names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28823))
+- Fix processing of compacted single-item JSON-LD collections ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28816))
+- Retry 401 errors on replies fetching ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/28788))
+- Fix `RecordNotUnique` errors in LinkCrawlWorker ([tribela](https://github.com/mastodon/mastodon/pull/28748))
+- Fix Mastodon not correctly processing HTTP Signatures with query strings ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28443), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28476))
+- Fix potential redirection loop of streaming endpoint ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28665))
+- Fix streaming API redirection ignoring the port of `streaming_api_base_url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28558))
+- Fix `Undo Announce` activity not being sent to non-follower authors ([MitarashiDango](https://github.com/mastodon/mastodon/pull/18482))
+- Fix `LinkCrawlWorker` error when encountering empty OEmbed response ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28268))
+
+### Security
+
+- Add rate-limit of TOTP authentication attempts at controller level ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28801))
+
+## [4.1.11] - 2023-12-04
+
+### Changed
+
+- Change GIF max matrix size error to explicitly mention GIF files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27927))
+- Change `Follow` activities delivery to bypass availability check ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/27586))
+- Change Content-Security-Policy to be tighter on media paths ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26889))
+
+### Fixed
+
+- Fix incoming status creation date not being restricted to standard ISO8601 ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27655), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28081))
+- Fix posts from force-sensitized accounts being able to trend ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27620))
+- Fix processing LDSigned activities from actors with unknown public keys ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27474))
+- Fix error and incorrect URLs in `/api/v1/accounts/:id/featured_tags` for remote accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27459))
+- Fix report processing notice not mentioning the report number when performing a custom action ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27442))
+- Fix some link anchors being recognized as hashtags ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27271), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27584))
+
+## [4.1.10] - 2023-10-10
+
+### Changed
+
+- Change some worker lock TTLs to be shorter-lived ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27246))
+- Change user archive export allowed period from 7 days to 6 days ([suddjian](https://github.com/mastodon/mastodon/pull/27200))
+
+### Fixed
+
+- Fix mentions being matched in some URL query strings ([mjankowski](https://github.com/mastodon/mastodon/pull/25656))
+- Fix multiple instances of the trend refresh scheduler sometimes running at once ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27253))
+- Fix importer returning negative row estimates ([jgillich](https://github.com/mastodon/mastodon/pull/27258))
+- Fix filtering audit log for entries about disabling 2FA ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27186))
+- Fix tIME chunk not being properly removed from PNG uploads ([TheEssem](https://github.com/mastodon/mastodon/pull/27111))
+- Fix inefficient queries in “Follows and followers” as well as several admin pages ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27116), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27306))
+
+## [4.1.9] - 2023-09-20
+
+### Fixed
+
+- Fix post translation erroring out ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26990))
+
+## [4.1.8] - 2023-09-19
+
+### Fixed
+
+- Fix post edits not being forwarded as expected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26936))
+- Fix moderator rights inconsistencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26729))
+- Fix crash when encountering invalid URL ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26814))
+- Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409))
+- Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500))
+- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608))
+
+### Security
+
+- Fix missing HTML sanitization in translation API (CVE-2023-42452)
+- Fix incorrect domain name normalization (CVE-2023-42451)
+
+## [4.1.7] - 2023-09-05
+
+### Changed
+
+- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028))
+
+### Fixed
+
+- **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392))
+- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237))
+- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727))
+
+## [4.1.6] - 2023-07-31
+
+### Fixed
+
+- Fix memory leak in streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26228))
+- Fix wrong filters sometimes applying in streaming ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26159), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26213), [renchap](https://github.com/mastodon/mastodon/pull/26233))
+- Fix incorrect connect timeout in outgoing requests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26116))
+
+## [4.1.5] - 2023-07-21
+
+### Added
+
+- Add check preventing Sidekiq workers from running with Makara configured ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25850))
+
+### Changed
+
+- Change request timeout handling to use a longer deadline ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26055))
+
+### Fixed
+
+- Fix moderation interface for remote instances with a .zip TLD ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
+- Fix remote accounts being possibly persisted to database with incomplete protocol values ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
+- Fix trending publishers table not rendering correctly on narrow screens ([vmstan](https://github.com/mastodon/mastodon/pull/25945))
+
+### Security
+
+- Fix CSP headers being unintentionally wide ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26105))
+
 ## [4.1.4] - 2023-07-07
 
 ### Fixed

Dockerfile

@@ -17,6 +17,7 @@ COPY Gemfile* package.json yarn.lock /opt/mastodon/
 
 # hadolint ignore=DL3008
 RUN apt-get update && \
+    apt-get -yq dist-upgrade && \
     apt-get install -y --no-install-recommends build-essential \
         ca-certificates \
         git \

Gemfile

@@ -158,3 +158,4 @@ gem 'concurrent-ruby', require: false
 gem 'connection_pool', require: false
 gem 'xorcist', '~> 1.1'
 gem 'cocoon', '~> 1.2'
+gem 'mail', '~> 2.8'

Gemfile.lock

@@ -10,40 +10,40 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7.4)
-      actionpack (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    actioncable (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7.4)
-      actionpack (= 6.1.7.4)
-      activejob (= 6.1.7.4)
-      activerecord (= 6.1.7.4)
-      activestorage (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    actionmailbox (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       mail (>= 2.7.1)
-    actionmailer (6.1.7.4)
-      actionpack (= 6.1.7.4)
-      actionview (= 6.1.7.4)
-      activejob (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    actionmailer (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      actionview (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7.4)
-      actionview (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    actionpack (6.1.7.8)
+      actionview (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7.4)
-      actionpack (= 6.1.7.4)
-      activerecord (= 6.1.7.4)
-      activestorage (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    actiontext (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7.4)
-      activesupport (= 6.1.7.4)
+    actionview (6.1.7.8)
+      activesupport (= 6.1.7.8)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -54,22 +54,22 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.8)
-    activejob (6.1.7.4)
-      activesupport (= 6.1.7.4)
+    activejob (6.1.7.8)
+      activesupport (= 6.1.7.8)
       globalid (>= 0.3.6)
-    activemodel (6.1.7.4)
-      activesupport (= 6.1.7.4)
-    activerecord (6.1.7.4)
-      activemodel (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
-    activestorage (6.1.7.4)
-      actionpack (= 6.1.7.4)
-      activejob (= 6.1.7.4)
-      activerecord (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    activemodel (6.1.7.8)
+      activesupport (= 6.1.7.8)
+    activerecord (6.1.7.8)
+      activemodel (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
+    activestorage (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7.4)
+    activesupport (6.1.7.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -105,6 +105,7 @@ GEM
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.5.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.2.0)
     bcrypt (3.1.17)
     better_errors (2.9.1)
       coderay (>= 1.0.0)
@@ -173,7 +174,7 @@ GEM
     cocoon (1.2.15)
     coderay (1.1.3)
     color_diff (0.1)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.3.0)
     cose (1.2.1)
       cbor (~> 0.5.9)
@@ -183,7 +184,7 @@ GEM
     crass (1.0.6)
     css_parser (1.12.0)
       addressable
-    date (3.3.3)
+    date (3.3.4)
     debug_inspector (1.0.0)
     devise (4.8.1)
       bcrypt (~> 3.0)
@@ -330,7 +331,7 @@ GEM
     jmespath (1.6.2)
     json (2.6.3)
     json-canonicalization (0.3.0)
-    json-jwt (1.15.3)
+    json-jwt (1.15.3.1)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
@@ -395,7 +396,7 @@ GEM
       net-smtp
     makara (0.5.1)
       activerecord (>= 5.2.0)
-    marcel (1.0.2)
+    marcel (1.0.4)
     mario-redis-lock (1.2.1)
       redis (>= 3.0.5)
     matrix (0.4.2)
@@ -404,28 +405,28 @@ GEM
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
-    mini_mime (1.1.2)
-    mini_portile2 (2.8.2)
+    mini_mime (1.1.5)
+    mini_portile2 (2.8.7)
     minitest (5.17.0)
     msgpack (1.6.0)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    net-imap (0.3.6)
+    net-imap (0.3.7)
       date
       net-protocol
     net-ldap (0.17.1)
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
     net-scp (4.0.0.rc1)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.3.3)
+    net-smtp (0.3.4)
       net-protocol
     net-ssh (7.0.1)
     nio4r (2.5.9)
-    nokogiri (1.14.5)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.16.6)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.2.8)
       activesupport (>= 4.2, < 7)
@@ -468,7 +469,7 @@ GEM
     parslet (2.0.0)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pg (1.4.5)
+    pg (1.4.6)
     pghero (3.1.0)
       activerecord (>= 6)
     pkg-config (1.5.1)
@@ -491,13 +492,13 @@ GEM
     pry-rails (0.3.9)
       pry (>= 0.10.4)
     public_suffix (5.0.1)
-    puma (5.6.5)
+    puma (5.6.8)
       nio4r (~> 2.0)
     pundit (2.3.0)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
-    racc (1.6.2)
-    rack (2.2.7)
+    racc (1.7.3)
+    rack (2.2.9)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
@@ -512,20 +513,20 @@ GEM
       rack
     rack-test (2.0.2)
       rack (>= 1.3)
-    rails (6.1.7.4)
-      actioncable (= 6.1.7.4)
-      actionmailbox (= 6.1.7.4)
-      actionmailer (= 6.1.7.4)
-      actionpack (= 6.1.7.4)
-      actiontext (= 6.1.7.4)
-      actionview (= 6.1.7.4)
-      activejob (= 6.1.7.4)
-      activemodel (= 6.1.7.4)
-      activerecord (= 6.1.7.4)
-      activestorage (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    rails (6.1.7.8)
+      actioncable (= 6.1.7.8)
+      actionmailbox (= 6.1.7.8)
+      actionmailer (= 6.1.7.8)
+      actionpack (= 6.1.7.8)
+      actiontext (= 6.1.7.8)
+      actionview (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activemodel (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       bundler (>= 1.15.0)
-      railties (= 6.1.7.4)
+      railties (= 6.1.7.8)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -541,9 +542,9 @@ GEM
       railties (>= 6.0.0, < 7)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (6.1.7.4)
-      actionpack (= 6.1.7.4)
-      activesupport (= 6.1.7.4)
+    railties (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -565,7 +566,8 @@ GEM
     responders (3.0.1)
       actionpack (>= 5.0)
       railties (>= 5.0)
-    rexml (3.2.5)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rotp (6.2.0)
     rpam2 (4.0.2)
     rqrcode (2.1.2)
@@ -634,7 +636,7 @@ GEM
       activerecord (>= 4.0.0)
       railties (>= 4.0.0)
     semantic_range (3.0.0)
-    sidekiq (6.5.8)
+    sidekiq (6.5.12)
       connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
       redis (>= 4.5.0, < 5)
@@ -645,7 +647,7 @@ GEM
       rufus-scheduler (~> 3.2)
       sidekiq (>= 4, < 7)
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (7.1.29)
+    sidekiq-unique-jobs (7.1.33)
       brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.5)
       redis (< 5.0)
@@ -663,7 +665,8 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     smart_properties (1.17.0)
-    sprockets (3.7.2)
+    sprockets (3.7.3)
+      base64
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.4.2)
@@ -679,6 +682,7 @@ GEM
       redlock (~> 1.0)
     strong_migrations (0.7.9)
       activerecord (>= 5)
+    strscan (3.1.0)
     swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
@@ -746,14 +750,14 @@ GEM
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     wisper (2.0.1)
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.8)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   ruby
@@ -816,6 +820,7 @@ DEPENDENCIES
   letter_opener_web (~> 2.0)
   link_header (~> 0.0)
   lograge (~> 0.12)
+  mail (~> 2.8)
   makara (~> 0.5)
   mario-redis-lock (~> 1.2)
   memory_profiler

SECURITY.md

@@ -10,8 +10,8 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 ## Supported Versions
 
-| Version | Supported |
-| ------- | ----------|
-| 4.0.x   | Yes       |
-| 3.5.x   | Yes       |
-| < 3.5   | No        |
+| Version | Supported        |
+| ------- | ---------------- |
+| 4.2.x   | Yes              |
+| 4.1.x   | Yes              |
+| < 4.1   | No               |

app/chewy/accounts_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class AccountsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: { refresh_interval: '30s' }, analysis: {
     analyzer: {
       content: {
@@ -38,6 +40,6 @@ class AccountsIndex < Chewy::Index
 
     field :following_count, type: 'long', value: ->(account) { account.following_count }
     field :followers_count, type: 'long', value: ->(account) { account.followers_count }
-    field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    field :last_status_at, type: 'date', value: ->(account) { clamp_date(account.last_status_at || account.created_at) }
   end
 end

app/chewy/concerns/datetime_clamping_concern.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DatetimeClampingConcern
+  extend ActiveSupport::Concern
+
+  MIN_ISO8601_DATETIME = '0000-01-01T00:00:00Z'.to_datetime.freeze
+  MAX_ISO8601_DATETIME = '9999-12-31T23:59:59Z'.to_datetime.freeze
+
+  class_methods do
+    def clamp_date(datetime)
+      datetime.clamp(MIN_ISO8601_DATETIME, MAX_ISO8601_DATETIME)
+    end
+  end
+end

app/chewy/tags_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class TagsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: { refresh_interval: '30s' }, analysis: {
     analyzer: {
       content: {
@@ -36,6 +38,6 @@ class TagsIndex < Chewy::Index
 
     field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
     field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts }
-    field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    field :last_status_at, type: 'date', value: ->(tag) { clamp_date(tag.last_status_at || tag.created_at) }
   end
 end

app/controllers/admin/account_actions_controller.rb

@@ -21,7 +21,7 @@ module Admin
       account_action.save!
 
       if account_action.with_report?
-        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: params[:report_id])
+        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: resource_params[:report_id])
       else
         redirect_to admin_account_path(@account.id)
       end

app/controllers/admin/domain_allows_controller.rb

@@ -25,6 +25,8 @@ class Admin::DomainAllowsController < Admin::BaseController
   def destroy
     authorize @domain_allow, :destroy?
     UnallowDomainService.new.call(@domain_allow)
+    log_action :destroy, @domain_allow
+
     redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
   end
 

app/controllers/admin/domain_blocks_controller.rb

@@ -37,7 +37,7 @@ module Admin
         @domain_block.errors.delete(:domain)
         render :new
       else
-        if existing_domain_block.present?
+        if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip)
           @domain_block = existing_domain_block
           @domain_block.update(resource_params)
         end

app/controllers/api/v1/admin/domain_blocks_controller.rb

@@ -19,10 +19,11 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
   def create
     authorize :domain_block, :create?
 
+    @domain_block = DomainBlock.new(resource_params)
     existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
-    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if existing_domain_block.present?
+    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if conflicts_with_existing_block?(@domain_block, existing_domain_block)
 
-    @domain_block = DomainBlock.create!(resource_params)
+    @domain_block.save!
     DomainBlockWorker.perform_async(@domain_block.id)
     log_action :create, @domain_block
     render json: @domain_block, serializer: REST::Admin::DomainBlockSerializer
@@ -55,6 +56,10 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
 
   private
 
+  def conflicts_with_existing_block?(domain_block, existing_domain_block)
+    existing_domain_block.present? && (existing_domain_block.domain == TagManager.instance.normalize_domain(domain_block.domain) || !domain_block.stricter_than?(existing_domain_block))
+  end
+
   def set_domain_blocks
     @domain_blocks = filtered_domain_blocks.order(id: :desc).to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end

app/controllers/api/v1/featured_tags/suggestions_controller.rb

@@ -12,6 +12,10 @@ class Api::V1::FeaturedTags::SuggestionsController < Api::BaseController
   private
 
   def set_recently_used_tags
-    @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10)
+    @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10)
+  end
+
+  def featured_tag_ids
+    current_account.featured_tags.pluck(:tag_id)
   end
 end

app/controllers/api/v1/scheduled_statuses_controller.rb

@@ -6,6 +6,7 @@ class Api::V1::ScheduledStatusesController < Api::BaseController
   before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, except: [:update, :destroy]
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:update, :destroy]
 
+  before_action :require_user!
   before_action :set_statuses, only: :index
   before_action :set_status, except: :index
 

app/controllers/api/v1/statuses/translations_controller.rb

@@ -4,6 +4,7 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController
   include Authorization
 
   before_action -> { doorkeeper_authorize! :read, :'read:statuses' }
+  before_action :require_user!
   before_action :set_status
   before_action :set_translation
 

app/controllers/api/v1/streaming_controller.rb

@@ -2,7 +2,7 @@
 
 class Api::V1::StreamingController < Api::BaseController
   def index
-    if Rails.configuration.x.streaming_api_base_url == request.host
+    if same_host?
       not_found
     else
       redirect_to streaming_api_url, status: 301
@@ -11,9 +11,16 @@ class Api::V1::StreamingController < Api::BaseController
 
   private
 
+  def same_host?
+    base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
+    request.host == base_url.host && request.port == (base_url.port || 80)
+  end
+
   def streaming_api_url
     Addressable::URI.parse(request.url).tap do |uri|
-      uri.host = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url).host
+      base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
+      uri.host = base_url.host
+      uri.port = base_url.port
     end.to_s
   end
 end

app/controllers/api/v1/timelines/public_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::PublicController < Api::BaseController
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
   before_action :require_user!, only: [:show], if: :require_auth?
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 

app/controllers/api/v1/timelines/tag_controller.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::TagController < Api::BaseController
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
+  before_action :require_user!, if: :require_auth?
   before_action :load_tag
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
@@ -15,6 +17,10 @@ class Api::V1::Timelines::TagController < Api::BaseController
 
   private
 
+  def require_auth?
+    !Setting.timeline_preview
+  end
+
   def load_tag
     @tag = Tag.find_normalized(params[:id])
   end

app/controllers/auth/omniauth_callbacks_controller.rb

@@ -5,7 +5,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
 
   def self.provides_callback_for(provider)
     define_method provider do
-      @user = User.find_for_oauth(request.env['omniauth.auth'], current_user)
+      @user = User.find_for_omniauth(request.env['omniauth.auth'], current_user)
 
       if @user.persisted?
         LoginActivity.create(
@@ -24,6 +24,9 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
         session["devise.#{provider}_data"] = request.env['omniauth.auth']
         redirect_to new_user_registration_url
       end
+    rescue ActiveRecord::RecordInvalid
+      flash[:alert] = I18n.t('devise.failure.omniauth_user_creation_failure') if is_navigational_format?
+      redirect_to new_user_session_url
     end
   end
 

app/controllers/auth/sessions_controller.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class Auth::SessionsController < Devise::SessionsController
+  include Redisable
+
+  MAX_2FA_ATTEMPTS_PER_HOUR = 10
+
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
@@ -136,9 +140,23 @@ class Auth::SessionsController < Devise::SessionsController
     session.delete(:attempt_user_updated_at)
   end
 
+  def clear_2fa_attempt_from_user(user)
+    redis.del(second_factor_attempts_key(user))
+  end
+
+  def check_second_factor_rate_limits(user)
+    attempts, = redis.multi do |multi|
+      multi.incr(second_factor_attempts_key(user))
+      multi.expire(second_factor_attempts_key(user), 1.hour)
+    end
+
+    attempts >= MAX_2FA_ATTEMPTS_PER_HOUR
+  end
+
   def on_authentication_success(user, security_measure)
     @on_authentication_success_called = true
 
+    clear_2fa_attempt_from_user(user)
     clear_attempt_from_session
 
     user.update_sign_in!(new_sign_in: true)
@@ -170,4 +188,8 @@ class Auth::SessionsController < Devise::SessionsController
       user_agent: request.user_agent
     )
   end
+
+  def second_factor_attempts_key(user)
+    "2fa_auth_attempts:#{user.id}:#{Time.now.utc.hour}"
+  end
 end

app/controllers/concerns/cache_concern.rb

@@ -28,29 +28,19 @@ module CacheConcern
     response.headers['Vary'] = public_fetch_mode? ? 'Accept' : 'Accept, Signature'
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection(raw, klass)
-    return raw unless klass.respond_to?(:with_includes)
+    return raw unless klass.respond_to?(:preload_cacheable_associations)
 
-    raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation)
-    return [] if raw.empty?
+    records = raw.to_a
 
-    cached_keys_with_value = Rails.cache.read_multi(*raw).transform_keys(&:id)
-    uncached_ids           = raw.map(&:id) - cached_keys_with_value.keys
+    klass.preload_cacheable_associations(records)
 
-    klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!)
-
-    unless uncached_ids.empty?
-      uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id)
-
-      uncached.each_value do |item|
-        Rails.cache.write(item, item)
-      end
-    end
-
-    raw.filter_map { |item| cached_keys_with_value[item.id] || uncached[item.id] }
+    records
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection_paginated_by_id(raw, klass, limit, options)
-    cache_collection raw.cache_ids.to_a_paginated_by_id(limit, options), klass
+    cache_collection raw.to_a_paginated_by_id(limit, options), klass
   end
 end

app/controllers/concerns/signature_verification.rb

@@ -91,14 +91,23 @@ module SignatureVerification
     raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if actor.nil?
 
     signature             = Base64.decode64(signature_params['signature'])
-    compare_signed_string = build_signed_string
+    compare_signed_string = build_signed_string(include_query_string: true)
 
     return actor unless verify_signature(actor, signature, compare_signed_string).nil?
 
+    # Compatibility quirk with older Mastodon versions
+    compare_signed_string = build_signed_string(include_query_string: false)
+    return actor unless verify_signature(actor, signature, compare_signed_string).nil?
+
     actor = stoplight_wrap_request { actor_refresh_key!(actor) }
 
     raise SignatureVerificationError, "Could not refresh public key #{signature_params['keyId']}" if actor.nil?
 
+    compare_signed_string = build_signed_string(include_query_string: true)
+    return actor unless verify_signature(actor, signature, compare_signed_string).nil?
+
+    # Compatibility quirk with older Mastodon versions
+    compare_signed_string = build_signed_string(include_query_string: false)
     return actor unless verify_signature(actor, signature, compare_signed_string).nil?
 
     fail_with! "Verification failed for #{actor.to_log_human_identifier} #{actor.uri} using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)", signed_string: compare_signed_string, signature: signature_params['signature']
@@ -177,16 +186,24 @@ module SignatureVerification
     nil
   end
 
-  def build_signed_string
+  def build_signed_string(include_query_string: true)
     signed_headers.map do |signed_header|
-      if signed_header == Request::REQUEST_TARGET
-        "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
-      elsif signed_header == '(created)'
+      case signed_header
+      when Request::REQUEST_TARGET
+        if include_query_string
+          "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.original_fullpath}"
+        else
+          # Current versions of Mastodon incorrectly omit the query string from the (request-target) pseudo-header.
+          # Therefore, temporarily support such incorrect signatures for compatibility.
+          # TODO: remove eventually some time after release of the fixed version
+          "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
+        end
+      when '(created)'
         raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank?
 
         "(created): #{signature_params['created']}"
-      elsif signed_header == '(expires)'
+      when '(expires)'
         raise SignatureVerificationError, 'Invalid pseudo-header (expires) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (expires) used but corresponding argument missing' if signature_params['expires'].blank?
 
@@ -246,7 +263,7 @@ module SignatureVerification
       stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, ''), suppress_errors: false) }
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
       account   = ActivityPub::TagManager.instance.uri_to_actor(key_id)
-      account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false, suppress_errors: false) }
+      account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, suppress_errors: false) }
       account
     end
   rescue Mastodon::PrivateNetworkAddressError => e

app/controllers/concerns/two_factor_authentication_concern.rb

@@ -65,6 +65,11 @@ module TwoFactorAuthenticationConcern
   end
 
   def authenticate_with_two_factor_via_otp(user)
+    if check_second_factor_rate_limits(user)
+      flash.now[:alert] = I18n.t('users.rate_limited')
+      return prompt_for_two_factor(user)
+    end
+
     if valid_otp_attempt?(user)
       on_authentication_success(user, :otp)
     else

app/controllers/oauth/authorized_applications_controller.rb

@@ -16,6 +16,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
 
   def destroy
     Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner)
+    Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner)
     super
   end
 

app/controllers/well_known/webfinger_controller.rb

@@ -20,7 +20,7 @@ module WellKnown
     def set_account
       username = username_from_resource
       @account = begin
-        if username == Rails.configuration.x.local_domain
+        if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain
           Account.representative
         else
           Account.find_local!(username)

app/helpers/jsonld_helper.rb

@@ -157,8 +157,8 @@ module JsonLdHelper
     end
   end
 
-  def fetch_resource(uri, id, on_behalf_of = nil)
-    unless id
+  def fetch_resource(uri, id_is_known, on_behalf_of = nil, request_options: {})
+    unless id_is_known
       json = fetch_resource_without_id_validation(uri, on_behalf_of)
 
       return if !json.is_a?(Hash) || unsupported_uri_scheme?(json['id'])
@@ -166,17 +166,29 @@ module JsonLdHelper
       uri = json['id']
     end
 
-    json = fetch_resource_without_id_validation(uri, on_behalf_of)
+    json = fetch_resource_without_id_validation(uri, on_behalf_of, request_options: request_options)
     json.present? && json['id'] == uri ? json : nil
   end
 
-  def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
+  def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false, request_options: {})
     on_behalf_of ||= Account.representative
 
-    build_request(uri, on_behalf_of).perform do |response|
+    build_request(uri, on_behalf_of, options: request_options).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
-      body_to_json(response.body_with_limit) if response.code == 200
+      body_to_json(response.body_with_limit) if response.code == 200 && valid_activitypub_content_type?(response)
+    end
+  end
+
+  def valid_activitypub_content_type?(response)
+    return true if response.mime_type == 'application/activity+json'
+
+    # When the mime type is `application/ld+json`, we need to check the profile,
+    # but `http.rb` does not parse it for us.
+    return false unless response.mime_type == 'application/ld+json'
+
+    response.headers[HTTP::Headers::CONTENT_TYPE]&.split(';')&.map(&:strip)&.any? do |str|
+      str.start_with?('profile="') && str[9...-1].split.include?('https://www.w3.org/ns/activitystreams')
     end
   end
 
@@ -206,8 +218,8 @@ module JsonLdHelper
     response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
   end
 
-  def build_request(uri, on_behalf_of = nil)
-    Request.new(:get, uri).tap do |request|
+  def build_request(uri, on_behalf_of = nil, options: {})
+    Request.new(:get, uri, **options).tap do |request|
       request.on_behalf_of(on_behalf_of) if on_behalf_of
       request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     end

app/lib/activitypub/activity.rb

@@ -153,7 +153,8 @@ class ActivityPub::Activity
   def fetch_remote_original_status
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
+
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
     elsif @object['url'].present?
       ::FetchRemoteStatusService.new.call(@object['url'], request_id: @options[:request_id])
     end

app/lib/activitypub/activity/create.rb

@@ -84,8 +84,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ApplicationRecord.transaction do
       @status = Status.create!(@params)
       attach_tags(@status)
+
+      # Delete status on zero follower user and nearly created account with include some replies
+      if like_a_spam?
+        @status = nil
+        raise ActiveRecord::Rollback
+      end
     end
 
+    return if @status.nil?
+
     resolve_thread(@status)
     fetch_replies(@status)
     distribute
@@ -103,7 +111,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
-    status
+    status if status&.account_id == @account.id
   end
 
   def process_status_params
@@ -445,4 +453,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   rescue
     nil
   end
+
+  SPAM_FILTER_MINIMUM_FOLLOWERS = ENV.fetch('SPAM_FILTER_MINIMUM_FOLLOWERS', 0).to_i
+  SPAM_FILTER_MINIMUM_CREATE_DAYS = ENV.fetch('SPAM_FILTER_MINIMUM_CREATE_DAYS', 1).to_i
+  SPAM_FILTER_MINIMUM_MENTIONS = ENV.fetch('SPAM_FILTER_MINIMUM_MENTIONS', 1).to_i
+  def like_a_spam?
+    (
+      !@status.account.local? &&
+      @status.account.followers_count <= SPAM_FILTER_MINIMUM_FOLLOWERS &&
+      @status.account.created_at > SPAM_FILTER_MINIMUM_CREATE_DAYS.day.ago &&
+      @mentions.count > SPAM_FILTER_MINIMUM_MENTIONS
+    )
+  end
 end

app/lib/activitypub/activity/flag.rb

@@ -16,7 +16,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
         @account,
         target_account,
         status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id),
-        comment: @json['content'] || '',
+        comment: report_comment,
         uri: report_uri
       )
     end
@@ -35,4 +35,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
   def report_uri
     @json['id'] unless @json['id'].nil? || invalid_origin?(@json['id'])
   end
+
+  def report_comment
+    (@json['content'] || '')[0...5000]
+  end
 end

app/lib/activitypub/activity/update.rb

@@ -28,6 +28,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
 
     return if @status.nil?
 
-    ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id])
+    ActivityPub::ProcessStatusUpdateService.new.call(@status, @json, @object, request_id: @options[:request_id])
   end
 end

app/lib/activitypub/linked_data_signature.rb

@@ -18,8 +18,8 @@ class ActivityPub::LinkedDataSignature
 
     return unless type == 'RsaSignature2017'
 
-    creator   = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
-    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false)
+    creator = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
+    creator = ActivityPub::FetchRemoteKeyService.new.call(creator_uri) if creator&.public_key.blank?
 
     return if creator.nil?
 
@@ -27,9 +27,9 @@ class ActivityPub::LinkedDataSignature
     document_hash  = hash(@json.without('signature'))
     to_be_verified = options_hash + document_hash
 
-    if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
-      creator
-    end
+    creator if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
+  rescue OpenSSL::PKey::RSAError
+    false
   end
 
   def sign!(creator, sign_with: nil)

app/lib/activitypub/parser/status_parser.rb

@@ -3,6 +3,8 @@
 class ActivityPub::Parser::StatusParser
   include JsonLdHelper
 
+  NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze
+
   # @param [Hash] json
   # @param [Hash] magic_values
   # @option magic_values [String] :followers_collection
@@ -53,7 +55,8 @@ class ActivityPub::Parser::StatusParser
   end
 
   def created_at
-    @object['published']&.to_datetime
+    datetime = @object['published']&.to_datetime
+    datetime if datetime.present? && (0..9999).cover?(datetime.year)
   rescue ArgumentError
     nil
   end
@@ -85,6 +88,13 @@ class ActivityPub::Parser::StatusParser
   end
 
   def language
+    lang = raw_language_code
+    lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang)
+  end
+
+  private
+
+  def raw_language_code
     if content_language_map?
       @object['contentMap'].keys.first
     elsif name_language_map?
@@ -94,8 +104,6 @@ class ActivityPub::Parser::StatusParser
     end
   end
 
-  private
-
   def audience_to
     as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) }
   end

app/lib/activitypub/tag_manager.rb

@@ -27,6 +27,8 @@ class ActivityPub::TagManager
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
+    when :flag
+      target.uri
     end
   end
 
@@ -41,6 +43,8 @@ class ActivityPub::TagManager
       account_status_url(target.account, target)
     when :emoji
       emoji_url(target)
+    when :flag
+      target.uri
     end
   end
 

app/lib/admin/account_statuses_filter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Admin::AccountStatusesFilter < AccountStatusesFilter
+  private
+
+  def blocked?
+    false
+  end
+end

app/lib/application_extension.rb

@@ -4,12 +4,34 @@ module ApplicationExtension
   extend ActiveSupport::Concern
 
   included do
+    include Redisable
+
     validates :name, length: { maximum: 60 }
     validates :website, url: true, length: { maximum: 2_000 }, if: :website?
     validates :redirect_uri, length: { maximum: 2_000 }
+
+    # The relationship used between Applications and AccessTokens is using
+    # dependent: delete_all, which means the ActiveRecord callback in
+    # AccessTokenExtension is not run, so instead we manually announce to
+    # streaming that these tokens are being deleted.
+    before_destroy :close_streaming_sessions, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
+
+  def close_streaming_sessions(resource_owner = nil)
+    # TODO: #28793 Combine into a single topic
+    payload = Oj.dump(event: :kill)
+    scope = access_tokens
+    scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil?
+    scope.in_batches do |tokens|
+      redis.pipelined do |pipeline|
+        tokens.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
+    end
+  end
 end

app/lib/importer/base_importer.rb

@@ -34,7 +34,9 @@ class Importer::BaseImporter
   # Estimate the amount of documents that would be indexed. Not exact!
   # @returns [Integer]
   def estimate!
-    ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples AS estimate FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['estimate'].to_i }
+    reltuples = ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['reltuples'].to_i }
+    # If the table has never yet been vacuumed or analyzed, reltuples contains -1
+    [reltuples, 0].max
   end
 
   # Import data from the database into the index

app/lib/link_details_extractor.rb

@@ -255,16 +255,21 @@ class LinkDetailsExtractor
   end
 
   def document
-    @document ||= Nokogiri::HTML(@html, nil, encoding)
+    @document ||= detect_encoding_and_parse_document
   end
 
-  def encoding
-    @encoding ||= begin
-      guess = detector.detect(@html, @html_charset)
-      guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  def detect_encoding_and_parse_document
+    [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
+      document = Nokogiri::HTML(@html, nil, encoding)
+      return document if document.to_s.valid_encoding?
     end
   end
 
+  def detect_encoding
+    guess = detector.detect(@html, @html_charset)
+    guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  end
+
   def detector
     @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
       detector.strip_tags = true

app/lib/plain_text_formatter.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 class PlainTextFormatter
-  include ActionView::Helpers::TextHelper
-
-  NEWLINE_TAGS_RE = /(<br \/>|<br>|<\/p>)+/.freeze
+  NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
 
   attr_reader :text, :local
 
@@ -18,7 +16,10 @@ class PlainTextFormatter
     if local?
       text
     else
-      html_entities.decode(strip_tags(insert_newlines)).chomp
+      node = Nokogiri::HTML.fragment(insert_newlines)
+      # Elements that are entirely removed with our Sanitize config
+      node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
+      node.text.chomp
     end
   end
 
@@ -27,8 +28,4 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
-
-  def html_entities
-    HTMLEntities.new
-  end
 end

app/lib/request.rb

@@ -4,14 +4,22 @@ require 'ipaddr'
 require 'socket'
 require 'resolv'
 
-# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
+# Use our own timeout class to avoid using HTTP.rb's timeout block
 # around the Socket#open method, since we use our own timeout blocks inside
 # that method
 #
 # Also changes how the read timeout behaves so that it is cumulative (closer
 # to HTTP::Timeout::Global, but still having distinct timeouts for other
 # operation types)
-class HTTP::Timeout::PerOperation
+class PerOperationWithDeadline < HTTP::Timeout::PerOperation
+  READ_DEADLINE = 30
+
+  def initialize(*args)
+    super
+
+    @read_deadline = options.fetch(:read_deadline, READ_DEADLINE)
+  end
+
   def connect(socket_class, host, port, nodelay = false)
     @socket = socket_class.open(host, port)
     @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
@@ -24,7 +32,7 @@ class HTTP::Timeout::PerOperation
 
   # Read data from the socket
   def readpartial(size, buffer = nil)
-    @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout
+    @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_deadline
 
     timeout = false
     loop do
@@ -33,7 +41,8 @@ class HTTP::Timeout::PerOperation
       return :eof if result.nil?
 
       remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0
+      raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout
+      raise HTTP::TimeoutError, "Read timed out after a total of #{@read_deadline} seconds" if remaining_time <= 0
       return result if result != :wait_readable
 
       # marking the socket for timeout. Why is this not being raised immediately?
@@ -46,7 +55,7 @@ class HTTP::Timeout::PerOperation
       # timeout. Else, the first timeout was a proper timeout.
       # This hack has to be done because io/wait#wait_readable doesn't provide a value for when
       # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
-      timeout = true unless @socket.to_io.wait_readable(remaining_time)
+      timeout = true unless @socket.to_io.wait_readable([remaining_time, @read_timeout].min)
     end
   end
 end
@@ -57,7 +66,7 @@ class Request
   # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
   # and 5s timeout on the TLS handshake, meaning the worst case should take
   # about 15s in total
-  TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
+  TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze
 
   include RoutingHelper
 
@@ -68,7 +77,9 @@ class Request
     @url         = Addressable::URI.parse(url).normalize
     @http_client = options.delete(:http_client)
     @allow_local = options.delete(:allow_local)
+    @full_path   = options.delete(:with_query_string)
     @options     = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
+    @options     = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT)
     @options     = @options.merge(proxy_url) if use_proxy?
     @headers     = {}
 
@@ -129,14 +140,14 @@ class Request
     end
 
     def http_client
-      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
+      HTTP.use(:auto_inflate).follow(max_hops: 3)
     end
   end
 
   private
 
   def set_common_headers!
-    @headers[REQUEST_TARGET]    = "#{@verb} #{@url.path}"
+    @headers[REQUEST_TARGET]    = request_target
     @headers['User-Agent']      = Mastodon::Version.user_agent
     @headers['Host']            = @url.host
     @headers['Date']            = Time.now.utc.httpdate
@@ -147,6 +158,14 @@ class Request
     @headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
   end
 
+  def request_target
+    if @url.query.nil? || !@full_path
+      "#{@verb} #{@url.path}"
+    else
+      "#{@verb} #{@url.path}?#{@url.query}"
+    end
+  end
+
   def signature
     algorithm = 'rsa-sha256'
     signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
@@ -275,11 +294,11 @@ class Request
         end
 
         until socks.empty?
-          _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
+          _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect_timeout])
 
           if available_socks.nil?
             socks.each(&:close)
-            raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
+            raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect_timeout]} seconds"
           end
 
           available_socks.each do |sock|

app/lib/search_query_transformer.rb

@@ -101,7 +101,7 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   rule(clause: subtree(:clause)) do
-    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    prefix   = clause[:prefix][:term].to_s.downcase if clause[:prefix]
     operator = clause[:operator]&.to_s
 
     if clause[:prefix]

app/lib/status_reach_finder.rb

@@ -16,28 +16,28 @@ class StatusReachFinder
   private
 
   def reached_account_inboxes
+    Account.where(id: reached_account_ids).inboxes
+  end
+
+  def reached_account_ids
     # When the status is a reblog, there are no interactions with it
     # directly, we assume all interactions are with the original one
 
     if @status.reblog?
-      []
+      [reblog_of_account_id]
     else
-      Account.where(id: reached_account_ids).inboxes
-    end
-  end
-
-  def reached_account_ids
-    [
-      replied_to_account_id,
-      reblog_of_account_id,
-      mentioned_account_ids,
-      reblogs_account_ids,
-      favourites_account_ids,
-      replies_account_ids,
-    ].tap do |arr|
-      arr.flatten!
-      arr.compact!
-      arr.uniq!
+      [
+        replied_to_account_id,
+        reblog_of_account_id,
+        mentioned_account_ids,
+        reblogs_account_ids,
+        favourites_account_ids,
+        replies_account_ids,
+      ].tap do |arr|
+        arr.flatten!
+        arr.compact!
+        arr.uniq!
+      end
     end
   end
 

app/lib/tag_manager.rb

@@ -7,18 +7,18 @@ class TagManager
   include RoutingHelper
 
   def web_domain?(domain)
-    domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.web_domain).zero?
+    domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.web_domain).zero?
   end
 
   def local_domain?(domain)
-    domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero?
+    domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.local_domain).zero?
   end
 
   def normalize_domain(domain)
     return if domain.nil?
 
     uri = Addressable::URI.new
-    uri.host = domain.gsub(/[\/]/, '')
+    uri.host = domain.delete_suffix('/')
     uri.normalized_host
   end
 
@@ -28,7 +28,7 @@ class TagManager
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
 
     TagManager.instance.web_domain?(domain)
-  rescue Addressable::URI::InvalidURIError
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
     false
   end
 end

app/lib/translation_service/deepl.rb

@@ -46,7 +46,7 @@ class TranslationService::DeepL < TranslationService
 
     raise UnexpectedResponseError unless json.is_a?(Hash)
 
-    Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
+    Translation.new(text: Sanitize.fragment(json.dig('translations', 0, 'text'), Sanitize::Config::MASTODON_STRICT), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
   rescue Oj::ParseError
     raise UnexpectedResponseError
   end

app/lib/translation_service/libre_translate.rb

@@ -37,7 +37,7 @@ class TranslationService::LibreTranslate < TranslationService
 
     raise UnexpectedResponseError unless json.is_a?(Hash)
 
-    Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate')
+    Translation.new(text: Sanitize.fragment(json['translatedText'], Sanitize::Config::MASTODON_STRICT), detected_source_language: source_language, provider: 'LibreTranslate')
   rescue Oj::ParseError
     raise UnexpectedResponseError
   end

app/lib/video_metadata_extractor.rb

@@ -22,7 +22,7 @@ class VideoMetadataExtractor
   private
 
   def ffmpeg_command_output
-    command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
+    command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
     command.run(path: @path, format: 'json', loglevel: 'fatal')
   end
 
@@ -43,6 +43,9 @@ class VideoMetadataExtractor
         @height      = video_stream[:height]
         @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
         @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
+        # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we
+        # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue.
+        @frame_rate ||= @r_frame_rate
       end
 
       if (audio_stream = audio_streams.first)

app/mailers/admin_mailer.rb

@@ -47,4 +47,13 @@ class AdminMailer < ApplicationMailer
       mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance)
     end
   end
+
+  def auto_close_registrations(recipient)
+    @me       = recipient
+    @instance = Rails.configuration.x.local_domain
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.auto_close_registrations.subject', instance: @instance)
+    end
+  end
 end

app/models/account.rb

@@ -61,9 +61,9 @@ class Account < ApplicationRecord
     trust_level
   )
 
-  USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
-  MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
-  URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
+  USERNAME_RE   = /[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?/i
+  MENTION_RE    = %r{(?<![=/[:word:]])@((#{USERNAME_RE})(?:@[[:word:].-]+[[:word:]]+)?)}i
+  URL_PREFIX_RE = %r{\Ahttp(s?)://[^/]+}
   USERNAME_ONLY_RE = /\A#{USERNAME_RE}\z/i
 
   include Attachmentable
@@ -112,10 +112,10 @@ class Account < ApplicationRecord
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) }
   scope :searchable, -> { without_unapproved.without_suspended.where(moved_to_account_id: nil) }
-  scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
+  scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).joins(:account_stat) }
   scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
-  scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
-  scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
+  scope :by_recent_status, -> { includes(:account_stat).merge(AccountStat.order('last_status_at DESC NULLS LAST')).references(:account_stat) }
+  scope :by_recent_sign_in, -> { order(Arel.sql('users.current_sign_in_at DESC NULLS LAST')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
   scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches("%.#{domain}"))) }
   scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) }

app/models/admin/action_log_filter.rb

@@ -38,7 +38,7 @@ class Admin::ActionLogFilter
     destroy_status: { target_type: 'Status', action: 'destroy' }.freeze,
     destroy_user_role: { target_type: 'UserRole', action: 'destroy' }.freeze,
     destroy_canonical_email_block: { target_type: 'CanonicalEmailBlock', action: 'destroy' }.freeze,
-    disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze,
+    disable_2fa_user: { target_type: 'User', action: 'disable_2fa' }.freeze,
     disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze,
     disable_user: { target_type: 'User', action: 'disable' }.freeze,
     enable_custom_emoji: { target_type: 'CustomEmoji', action: 'enable' }.freeze,

app/models/admin/status_batch_action.rb

@@ -140,6 +140,6 @@ class Admin::StatusBatchAction
   end
 
   def allowed_status_ids
-    AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
+    Admin::AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
   end
 end

app/models/concerns/account_avatar.rb

@@ -18,7 +18,7 @@ module AccountAvatar
 
   included do
     # Avatar upload
-    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail]
+    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :avatar, less_than: LIMIT
     remotable_attachment :avatar, LIMIT, suppress_errors: false

app/models/concerns/account_header.rb

@@ -19,7 +19,7 @@ module AccountHeader
 
   included do
     # Header upload
-    has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail]
+    has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :header, less_than: LIMIT
     remotable_attachment :header, LIMIT, suppress_errors: false

app/models/concerns/account_interactions.rb

@@ -187,7 +187,7 @@ module AccountInteractions
   end
 
   def unblock_domain!(other_domain)
-    block = domain_blocks.find_by(domain: other_domain)
+    block = domain_blocks.find_by(domain: normalized_domain(other_domain))
     block&.destroy
   end
 
@@ -299,4 +299,8 @@ module AccountInteractions
   def remove_potential_friendship(other_account)
     PotentialFriendshipTracker.remove(id, other_account.id)
   end
+
+  def normalized_domain(domain)
+    TagManager.instance.normalize_domain(domain)
+  end
 end

app/models/concerns/attachmentable.rb

@@ -52,9 +52,13 @@ module Attachmentable
     return if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank?
 
     width, height = FastImage.size(attachment.queued_for_write[:original].path)
-    matrix_limit  = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT
+    return unless width.present? && height.present?
 
-    raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit)
+    if attachment.content_type == 'image/gif' && width * height > GIF_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} GIF files are not supported"
+    elsif width * height > MAX_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported"
+    end
   end
 
   def appropriate_extension(attachment)

app/models/concerns/cacheable.rb

@@ -14,6 +14,10 @@ module Cacheable
       includes(@cache_associated)
     end
 
+    def preload_cacheable_associations(records)
+      ActiveRecord::Associations::Preloader.new.preload(records, @cache_associated)
+    end
+
     def cache_ids
       select(:id, :updated_at)
     end

app/models/concerns/omniauthable.rb

@@ -19,17 +19,18 @@ module Omniauthable
   end
 
   class_methods do
-    def find_for_oauth(auth, signed_in_resource = nil)
+    def find_for_omniauth(auth, signed_in_resource = nil)
       # EOLE-SSO Patch
       auth.uid = (auth.uid[0][:uid] || auth.uid[0][:user]) if auth.uid.is_a? Hashie::Array
-      identity = Identity.find_for_oauth(auth)
+      identity = Identity.find_for_omniauth(auth)
 
       # If a signed_in_resource is provided it always overrides the existing user
       # to prevent the identity being locked with accidentally created accounts.
       # Note that this may leave zombie accounts (with no associated identity) which
       # can be cleaned up at a later date.
       user   = signed_in_resource || identity.user
-      user ||= create_for_oauth(auth)
+      user ||= reattach_for_auth(auth)
+      user ||= create_for_auth(auth)
 
       if identity.user.nil?
         identity.user = user
@@ -39,19 +40,35 @@ module Omniauthable
       user
     end
 
-    def create_for_oauth(auth)
-      # Check if the user exists with provided email. If no email was provided,
+    private
+
+    def reattach_for_auth(auth)
+      # If allowed, check if a user exists with the provided email address,
+      # and return it if they does not have an associated identity with the
+      # current authentication provider.
+
+      # This can be used to provide a choice of alternative auth providers
+      # or provide smooth gradual transition between multiple auth providers,
+      # but this is discouraged because any insecure provider will put *all*
+      # local users at risk, regardless of which provider they registered with.
+
+      return unless ENV['ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH'] == 'true'
+
+      email, email_is_verified = email_from_auth(auth)
+      return unless email_is_verified
+
+      user = User.find_by(email: email)
+      return if user.nil? || Identity.exists?(provider: auth.provider, user_id: user.id)
+
+      user
+    end
+
+    def create_for_auth(auth)
+      # Create a user for the given auth params. If no email was provided,
       # we assign a temporary email and ask the user to verify it on
       # the next step via Auth::SetupController.show
 
-      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
-      assume_verified   = strategy&.security&.assume_email_is_verified
-      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
-      email             = auth.info.verified_email || auth.info.email
-
-      user = User.find_by(email: email) if email_is_verified
-
-      return user unless user.nil?
+      email, email_is_verified = email_from_auth(auth)
 
       user = User.new(user_params_from_auth(email, auth))
 
@@ -68,7 +85,14 @@ module Omniauthable
       user
     end
 
-    private
+    def email_from_auth(auth)
+      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
+      assume_verified   = strategy&.security&.assume_email_is_verified
+      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
+      email             = auth.info.verified_email || auth.info.email
+
+      [email, email_is_verified]
+    end
 
     def user_params_from_auth(email, auth)
       {

app/models/custom_emoji.rb

@@ -37,7 +37,7 @@ class CustomEmoji < ApplicationRecord
   belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
 
-  has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false
+  has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' } }, validate_media_type: false
 
   before_validation :downcase_domain
 

app/models/feed.rb

@@ -28,7 +28,7 @@ class Feed
       unhydrated = redis.zrangebyscore(key, "(#{min_id}", "(#{max_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
     end
 
-    Status.where(id: unhydrated).cache_ids
+    Status.where(id: unhydrated)
   end
 
   def key

app/models/identity.rb

@@ -16,7 +16,7 @@ class Identity < ApplicationRecord
   validates :uid, presence: true, uniqueness: { scope: :provider }
   validates :provider, presence: true
 
-  def self.find_for_oauth(auth)
+  def self.find_for_omniauth(auth)
     find_or_create_by(uid: auth.uid, provider: auth.provider)
   end
 end

app/models/media_attachment.rb

@@ -169,7 +169,7 @@ class MediaAttachment < ApplicationRecord
   }.freeze
 
   GLOBAL_CONVERT_OPTIONS = {
-    all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date',
+    all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp',
   }.freeze
 
   belongs_to :account,          inverse_of: :media_attachments, optional: true

app/models/preview_card.rb

@@ -50,7 +50,7 @@ class PreviewCard < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy
 
-  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date' }, validate_media_type: false
+  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, validate_media_type: false
 
   validates :url, presence: true, uniqueness: true
   validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES

app/models/preview_card_provider.rb

@@ -25,7 +25,7 @@ class PreviewCardProvider < ApplicationRecord
 
   validates :domain, presence: true, uniqueness: true, domain: true
 
-  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false
+  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' } }, validate_media_type: false
   validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
   remotable_attachment :icon, LIMIT
 

app/models/public_feed.rb

@@ -29,7 +29,7 @@ class PublicFeed
     scope.merge!(media_only_scope) if media_only?
     scope.merge!(language_scope) if account&.chosen_languages.present?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private

app/models/relationship_filter.rb

@@ -60,13 +60,13 @@ class RelationshipFilter
   def relationship_scope(value)
     case value
     when 'following'
-      account.following.eager_load(:account_stat).reorder(nil)
+      account.following.includes(:account_stat).reorder(nil)
     when 'followed_by'
-      account.followers.eager_load(:account_stat).reorder(nil)
+      account.followers.includes(:account_stat).reorder(nil)
     when 'mutual'
-      account.followers.eager_load(:account_stat).reorder(nil).merge(Account.where(id: account.following))
+      account.followers.includes(:account_stat).reorder(nil).merge(Account.where(id: account.following))
     when 'invited'
-      Account.joins(user: :invite).merge(Invite.where(user: account.user)).eager_load(:account_stat).reorder(nil)
+      Account.joins(user: :invite).merge(Invite.where(user: account.user)).includes(:account_stat).reorder(nil)
     else
       raise Mastodon::InvalidParameterError, "Unknown relationship: #{value}"
     end
@@ -112,7 +112,7 @@ class RelationshipFilter
   def activity_scope(value)
     case value
     when 'dormant'
-      AccountStat.where(last_status_at: nil).or(AccountStat.where(AccountStat.arel_table[:last_status_at].lt(1.month.ago)))
+      Account.joins(:account_stat).where(account_stat: { last_status_at: [nil, ...1.month.ago] })
     else
       raise Mastodon::InvalidParameterError, "Unknown activity: #{value}"
     end

app/models/report.rb

@@ -39,7 +39,10 @@ class Report < ApplicationRecord
   scope :resolved,   -> { where.not(action_taken_at: nil) }
   scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) }
 
-  validates :comment, length: { maximum: 1_000 }
+  # A report is considered local if the reporter is local
+  delegate :local?, to: :account
+
+  validates :comment, length: { maximum: 1_000 }, if: :local?
   validates :rule_ids, absence: true, unless: :violation?
 
   validate :validate_rule_ids
@@ -50,10 +53,6 @@ class Report < ApplicationRecord
     violation: 2_000,
   }
 
-  def local?
-    false # Force uri_for to use uri attribute
-  end
-
   before_validation :set_uri, only: :create
 
   after_create_commit :trigger_webhooks

app/models/site_upload.rb

@@ -40,7 +40,7 @@ class SiteUpload < ApplicationRecord
     mascot: {}.freeze,
   }.freeze
 
-  has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
+  has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
 
   validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/
   validates :file, presence: true

app/models/status.rb

@@ -361,26 +361,6 @@ class Status < ApplicationRecord
       StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |p, h| h[p.status_id] = true }
     end
 
-    def reload_stale_associations!(cached_items)
-      account_ids = []
-
-      cached_items.each do |item|
-        account_ids << item.account_id
-        account_ids << item.reblog.account_id if item.reblog?
-      end
-
-      account_ids.uniq!
-
-      return if account_ids.empty?
-
-      accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
-
-      cached_items.each do |item|
-        item.account = accounts[item.account_id]
-        item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
-      end
-    end
-
     def from_text(text)
       return [] if text.blank?
 

app/models/status_edit.rb

@@ -41,7 +41,7 @@ class StatusEdit < ApplicationRecord
   default_scope { order(id: :asc) }
 
   delegate :local?, :application, :edited?, :edited_at,
-           :discarded?, :visibility, to: :status
+           :discarded?, :visibility, :language, to: :status
 
   def emojis
     return @emojis if defined?(@emojis)

app/models/tag.rb

@@ -33,7 +33,7 @@ class Tag < ApplicationRecord
   HASTAG_LAST_SEQUENCE = '([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_NAME_PAT = "#{HASHTAG_FIRST_SEQUENCE}|#{HASTAG_LAST_SEQUENCE}"
 
-  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_PAT})/i
+  HASHTAG_RE = %r{(?<![=/)\w])#(#{HASHTAG_NAME_PAT})}i
   HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
   HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/
 

app/models/tag_feed.rb

@@ -33,7 +33,7 @@ class TagFeed < PublicFeed
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private

app/models/trends/statuses.rb

@@ -91,7 +91,7 @@ class Trends::Statuses < Trends::Base
   private
 
   def eligible?(status)
-    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
+    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && !status.account.sensitized? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
   end
 
   def calculate_scores(statuses, at_time)

app/models/user.rb

@@ -94,6 +94,9 @@ class User < ApplicationRecord
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
+
+  validates :email, presence: true, email_address: true
+
   validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
@@ -400,6 +403,16 @@ class User < ApplicationRecord
     Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch|
       batch.update_all(revoked_at: Time.now.utc)
       Web::PushSubscription.where(access_token_id: batch).delete_all
+
+      # Revoke each access token for the Streaming API, since `update_all``
+      # doesn't trigger ActiveRecord Callbacks:
+      # TODO: #28793 Combine into a single topic
+      payload = Oj.dump(event: :kill)
+      redis.pipelined do |pipeline|
+        batch.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
     end
   end
 

app/policies/admin/status_policy.rb

@@ -12,7 +12,7 @@ class Admin::StatusPolicy < ApplicationPolicy
   end
 
   def show?
-    role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported?)
+    role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported? || viewable_through_normal_policy?)
   end
 
   def destroy?
@@ -26,4 +26,10 @@ class Admin::StatusPolicy < ApplicationPolicy
   def review?
     role.can?(:manage_taxonomies)
   end
+
+  private
+
+  def viewable_through_normal_policy?
+    StatusPolicy.new(current_account, record, @preloaded_relations).show?
+  end
 end

app/policies/backup_policy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class BackupPolicy < ApplicationPolicy
-  MIN_AGE = 1.week
+  MIN_AGE = 6.days
 
   def create?
     user_signed_in? && current_user.backups.where('created_at >= ?', MIN_AGE.ago).count.zero?

app/serializers/manifest_serializer.rb

@@ -16,11 +16,18 @@ class ManifestSerializer < ActiveModel::Serializer
     512
   ).freeze
 
-  attributes :name, :short_name,
+  attributes :id, :name, :short_name,
              :icons, :theme_color, :background_color,
              :display, :start_url, :scope,
              :share_target, :shortcuts
 
+  def id
+    # This is set to `/home` because that was the old value of `start_url` and
+    # thus the fallback ID computed by Chrome:
+    # https://developer.chrome.com/blog/pwa-manifest-id/
+    '/home'
+  end
+
   def name
     object.title
   end
@@ -53,7 +60,7 @@ class ManifestSerializer < ActiveModel::Serializer
   end
 
   def start_url
-    '/home'
+    '/'
   end
 
   def scope

app/serializers/rest/admin/domain_block_serializer.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
 class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer
-  attributes :id, :domain, :created_at, :severity,
+  attributes :id, :domain, :digest, :created_at, :severity,
              :reject_media, :reject_reports,
              :private_comment, :public_comment, :obfuscate
 
   def id
     object.id.to_s
   end
+
+  def digest
+    object.domain_digest
+  end
 end

app/serializers/rest/featured_tag_serializer.rb

@@ -10,7 +10,9 @@ class REST::FeaturedTagSerializer < ActiveModel::Serializer
   end
 
   def url
-    short_account_tag_url(object.account, object.tag)
+    # The path is hardcoded because we have to deal with both local and
+    # remote users, which are different routes
+    account_with_domain_url(object.account, "tagged/#{object.tag.to_param}")
   end
 
   def name

app/services/activitypub/fetch_featured_collection_service.rb

@@ -23,9 +23,9 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 

app/services/activitypub/fetch_remote_account_service.rb

@@ -2,7 +2,7 @@
 
 class ActivityPub::FetchRemoteAccountService < ActivityPub::FetchRemoteActorService
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
+  def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
     actor = super
     return actor if actor.nil? || actor.is_a?(Account)
 

app/services/activitypub/fetch_remote_actor_service.rb

@@ -10,15 +10,15 @@ class ActivityPub::FetchRemoteActorService < BaseService
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
 
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
+  def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
     return if domain_not_allowed?(uri)
     return ActivityPub::TagManager.instance.uri_to_actor(uri) if ActivityPub::TagManager.instance.local_uri?(uri)
 
     @json = begin
       if prefetched_body.nil?
-        fetch_resource(uri, id)
+        fetch_resource(uri, true)
       else
-        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+        body_to_json(prefetched_body, compare_id: uri)
       end
     rescue Oj::ParseError
       raise Error, "Error parsing JSON-LD document #{uri}"

app/services/activitypub/fetch_remote_key_service.rb

@@ -6,23 +6,10 @@ class ActivityPub::FetchRemoteKeyService < BaseService
   class Error < StandardError; end
 
   # Returns actor that owns the key
-  def call(uri, id: true, prefetched_body: nil, suppress_errors: true)
+  def call(uri, suppress_errors: true)
     raise Error, 'No key URI given' if uri.blank?
 
-    if prefetched_body.nil?
-      if id
-        @json = fetch_resource_without_id_validation(uri)
-        if actor_type?
-          @json = fetch_resource(@json['id'], true)
-        elsif uri != @json['id']
-          raise Error, "Fetched URI #{uri} has wrong id #{@json['id']}"
-        end
-      else
-        @json = fetch_resource(uri, id)
-      end
-    else
-      @json = body_to_json(prefetched_body, compare_id: id ? uri : nil)
-    end
+    @json = fetch_resource(uri, false)
 
     raise Error, "Unable to fetch key JSON at #{uri}" if @json.nil?
     raise Error, "Unsupported JSON-LD context for document #{uri}" unless supported_context?(@json)

app/services/activitypub/fetch_remote_poll_service.rb

@@ -8,6 +8,6 @@ class ActivityPub::FetchRemotePollService < BaseService
 
     return unless supported_context?(json)
 
-    ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json)
+    ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json, json)
   end
 end

app/services/activitypub/fetch_remote_status_service.rb

@@ -7,13 +7,13 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   DISCOVERIES_PER_REQUEST = 1000
 
   # Should be called when uri has already been checked for locality
-  def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
+  def call(uri, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
     @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}"
     @json = begin
       if prefetched_body.nil?
-        fetch_resource(uri, id, on_behalf_of)
+        fetch_resource(uri, true, on_behalf_of)
       else
-        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+        body_to_json(prefetched_body, compare_id: uri)
       end
     end
 
@@ -63,7 +63,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
   def account_from_uri(uri)
     actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
-    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true, request_id: @request_id) if actor.nil? || actor.possibly_stale?
+    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, request_id: @request_id) if actor.nil? || actor.possibly_stale?
     actor
   end
 

app/services/activitypub/fetch_replies_service.rb

@@ -26,9 +26,9 @@ class ActivityPub::FetchRepliesService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 
@@ -36,7 +36,21 @@ class ActivityPub::FetchRepliesService < BaseService
     return collection_or_uri if collection_or_uri.is_a?(Hash)
     return unless @allow_synchronous_requests
     return if invalid_origin?(collection_or_uri)
-    fetch_resource_without_id_validation(collection_or_uri, nil, true)
+
+    # NOTE: For backward compatibility reasons, Mastodon signs outgoing
+    # queries incorrectly by default.
+    #
+    # While this is relevant for all URLs with query strings, this is
+    # the only code path where this happens in practice.
+    #
+    # Therefore, retry with correct signatures if this fails.
+    begin
+      fetch_resource_without_id_validation(collection_or_uri, nil, true)
+    rescue Mastodon::UnexpectedResponseError => e
+      raise unless e.response && e.response.code == 401 && Addressable::URI.parse(collection_or_uri).query.present?
+
+      fetch_resource_without_id_validation(collection_or_uri, nil, true, request_options: { with_query_string: true })
+    end
   end
 
   def filtered_replies

app/services/activitypub/process_account_service.rb

@@ -76,6 +76,9 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.suspended_at      = domain_block.created_at if auto_suspend?
     @account.suspension_origin = :local if auto_suspend?
     @account.silenced_at       = domain_block.created_at if auto_silence?
+
+    set_immediate_protocol_attributes!
+
     @account.save
   end
 
@@ -196,10 +199,15 @@ class ActivityPub::ProcessAccountService < BaseService
     value = first_of_value(@json[key])
 
     return if value.nil?
-    return value['url'] if value.is_a?(Hash)
 
-    image = fetch_resource_without_id_validation(value)
-    image['url'] if image
+    if value.is_a?(String)
+      value = fetch_resource_without_id_validation(value)
+      return if value.nil?
+    end
+
+    value = first_of_value(value['url']) if value.is_a?(Hash) && value['type'] == 'Image'
+    value = value['href'] if value.is_a?(Hash)
+    value if value.is_a?(String)
   end
 
   def public_key
@@ -271,7 +279,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def moved_account
     account   = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account)
-    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true, request_id: @options[:request_id])
+    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], break_on_redirect: true, request_id: @options[:request_id])
     account
   end
 

app/services/activitypub/process_status_update_service.rb

@@ -5,10 +5,11 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   include Redisable
   include Lockable
 
-  def call(status, json, request_id: nil)
+  def call(status, activity_json, object_json, request_id: nil)
     raise ArgumentError, 'Status has unsaved changes' if status.changed?
 
-    @json                      = json
+    @activity_json             = activity_json
+    @json                      = object_json
     @status_parser             = ActivityPub::Parser::StatusParser.new(@json)
     @uri                       = @status_parser.uri
     @status                    = status
@@ -308,6 +309,6 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   end
 
   def forwarder
-    @forwarder ||= ActivityPub::Forwarder.new(@account, @json, @status)
+    @forwarder ||= ActivityPub::Forwarder.new(@account, @activity_json, @status)
   end
 end

app/services/activitypub/synchronize_followers_service.rb

@@ -59,9 +59,9 @@ class ActivityPub::SynchronizeFollowersService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 

app/services/fetch_link_card_service.rb

@@ -15,6 +15,9 @@ class FetchLinkCardService < BaseService
     )
   }iox
 
+  # URL size limit to safely store in PosgreSQL's unique indexes
+  BYTESIZE_LIMIT = 2692
+
   def call(status)
     @status       = status
     @original_url = parse_urls
@@ -85,7 +88,7 @@ class FetchLinkCardService < BaseService
 
   def bad_url?(uri)
     # Avoid local instance URLs and invalid URLs
-    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
+    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT
   end
 
   def mention_link?(anchor)

app/services/fetch_oembed_service.rb

@@ -100,7 +100,7 @@ class FetchOEmbedService
   end
 
   def validate(oembed)
-    oembed if oembed[:version].to_s == '1.0' && oembed[:type].present?
+    oembed if oembed.present? && oembed[:version].to_s == '1.0' && oembed[:type].present?
   end
 
   def html

app/services/fetch_resource_service.rb

@@ -43,11 +43,19 @@ class FetchResourceService < BaseService
     @response_code = response.code
     return nil if response.code != 200
 
-    if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
+    if valid_activitypub_content_type?(response)
       body = response.body_with_limit
       json = body_to_json(body)
 
-      [json['id'], { prefetched_body: body, id: true }] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json))
+      return unless supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json))
+
+      if json['id'] != @url
+        return if terminal
+
+        return process(json['id'], terminal: true)
+      end
+
+      [@url, { prefetched_body: body }]
     elsif !terminal
       link_header = response['Link'] && parse_link_header(response)
 

app/services/follow_service.rb

@@ -71,7 +71,7 @@ class FollowService < BaseService
     if @target_account.local?
       LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
     elsif @target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url)
+      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, { 'bypass_availability' => true })
     end
 
     follow_request

app/services/keys/query_service.rb

@@ -69,7 +69,7 @@ class Keys::QueryService < BaseService
 
     return if json['items'].blank?
 
-    @devices = json['items'].map do |device|
+    @devices = as_array(json['items']).map do |device|
       Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim'])
     end
   rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e

app/services/notify_service.rb

@@ -62,16 +62,17 @@ class NotifyService < BaseService
           LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
           WHERE s.id = :id
         UNION ALL
-          SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1
-          FROM ancestors st
-          JOIN statuses s ON s.id = st.in_reply_to_id
-          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
-          WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit
+          SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1
+          FROM ancestors
+          JOIN statuses s ON s.id = ancestors.in_reply_to_id
+          /* early exit if we already have a mention matching our requirements */
+          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id
+          WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit
       )
       SELECT COUNT(*)
-      FROM ancestors st
-      JOIN statuses s ON s.id = st.id
-      WHERE st.mention_id IS NOT NULL AND s.visibility = 3
+      FROM ancestors
+      JOIN statuses s ON s.id = ancestors.id
+      WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3
     SQL
   end