Bump version to v4.1.15

* Fix insufficient Content-Type checking of fetched ActivityStreams objects

* Allow JSON-LD documents with multiple profiles

.circleci/config.yml

@@ -1,225 +0,0 @@
-version: 2.1
-
-orbs:
-  ruby: circleci/ruby@2.0.0
-  node: circleci/node@5.0.3
-
-executors:
-  default:
-    parameters:
-      ruby-version:
-        type: string
-    docker:
-      - image: cimg/ruby:<< parameters.ruby-version >>
-        environment:
-          BUNDLE_JOBS: 3
-          BUNDLE_RETRY: 3
-          CONTINUOUS_INTEGRATION: true
-          DB_HOST: localhost
-          DB_USER: root
-          DISABLE_SIMPLECOV: true
-          RAILS_ENV: test
-      - image: cimg/postgres:14.5
-        environment:
-          POSTGRES_USER: root
-          POSTGRES_HOST_AUTH_METHOD: trust
-      - image: cimg/redis:7.0
-
-commands:
-  install-system-dependencies:
-    steps:
-      - run:
-          name: Install system dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y libicu-dev libidn11-dev
-  install-ruby-dependencies:
-    parameters:
-      ruby-version:
-        type: string
-    steps:
-      - run:
-          command: |
-            bundle config clean 'true'
-            bundle config frozen 'true'
-            bundle config without 'development production'
-          name: Set bundler settings
-      - ruby/install-deps:
-          bundler-version: '2.3.26'
-          key: ruby<< parameters.ruby-version >>-gems-v1
-  wait-db:
-    steps:
-      - run:
-          command: dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m
-          name: Wait for PostgreSQL and Redis
-
-jobs:
-  build:
-    docker:
-      - image: cimg/ruby:3.0-node
-        environment:
-          RAILS_ENV: test
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - node/install-packages:
-          cache-version: v1
-          pkg-manager: yarn
-      - run:
-          command: |
-            export NODE_OPTIONS=--openssl-legacy-provider
-            ./bin/rails assets:precompile
-          name: Precompile assets
-      - persist_to_workspace:
-          paths:
-            - public/assets
-            - public/packs-test
-          root: .
-
-  test:
-    parameters:
-      ruby-version:
-        type: string
-    executor:
-      name: default
-      ruby-version: << parameters.ruby-version >>
-    environment:
-      ALLOW_NOPAM: true
-      PAM_ENABLED: true
-      PAM_DEFAULT_SERVICE: pam_test
-      PAM_CONTROLLED_SERVICE: pam_test_controlled
-    parallelism: 4
-    steps:
-      - checkout
-      - install-system-dependencies
-      - run:
-          command: sudo apt-get install -y ffmpeg imagemagick libpam-dev
-          name: Install additional system dependencies
-      - run:
-          command: bundle config with 'pam_authentication'
-          name: Enable PAM authentication
-      - install-ruby-dependencies:
-          ruby-version: << parameters.ruby-version >>
-      - attach_workspace:
-          at: .
-      - wait-db
-      - run:
-          command: ./bin/rails db:create db:schema:load db:seed
-          name: Load database schema
-      - ruby/rspec-test
-
-  test-migrations:
-    executor:
-      name: default
-      ruby-version: '3.0'
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - wait-db
-      - run:
-          command: ./bin/rails db:create
-          name: Create database
-      - run:
-          command: ./bin/rails db:migrate VERSION=20171010025614
-          name: Run migrations up to v2.0.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180514140000
-          name: Run migrations up to v2.4.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180707154237
-          name: Run migrations up to v2.4.3
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4_3
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all remaining migrations
-      - run:
-          command: ./bin/rails tests:migrations:check_database
-          name: Check migration result
-
-  test-two-step-migrations:
-    executor:
-      name: default
-      ruby-version: '3.0'
-    steps:
-      - checkout
-      - install-system-dependencies
-      - install-ruby-dependencies:
-          ruby-version: '3.0'
-      - wait-db
-      - run:
-          command: ./bin/rails db:create
-          name: Create database
-      - run:
-          command: ./bin/rails db:migrate VERSION=20171010025614
-          name: Run migrations up to v2.0.0
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180514140000
-          name: Run pre-deployment migrations up to v2.4.0
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate VERSION=20180707154237
-          name: Run migrations up to v2.4.3
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails tests:migrations:populate_v2_4_3
-          name: Populate database with test data
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all remaining pre-deployment migrations
-          environment:
-            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-      - run:
-          command: ./bin/rails db:migrate
-          name: Run all post-deployment migrations
-      - run:
-          command: ./bin/rails tests:migrations:check_database
-          name: Check migration result
-
-workflows:
-  version: 2
-  build-and-test:
-    jobs:
-      - build
-      - test:
-          matrix:
-            parameters:
-              ruby-version:
-                - '2.7'
-                - '3.0'
-          name: test-ruby<< matrix.ruby-version >>
-          requires:
-            - build
-      - test-migrations:
-          requires:
-            - build
-      - test-two-step-migrations:
-          requires:
-            - build
-      - node/run:
-          cache-version: v1
-          name: test-webui
-          pkg-manager: yarn
-          requires:
-            - build
-          version: '16.18'
-          yarn-run: test:jest

.github/workflows/build-container-image.yml

@@ -0,0 +1,92 @@
+on:
+  workflow_call:
+    inputs:
+      platforms:
+        required: true
+        type: string
+      cache:
+        type: boolean
+        default: true
+      use_native_arm64_builder:
+        type: boolean
+      push_to_images:
+        type: string
+      flavor:
+        type: string
+      tags:
+        type: string
+      labels:
+        type: string
+
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: docker/setup-qemu-action@v2
+        if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder
+
+      - uses: docker/setup-buildx-action@v2
+        id: buildx
+        if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }}
+
+      - name: Start a local Docker Builder
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        run: |
+          docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234
+
+      - uses: docker/setup-buildx-action@v2
+        id: buildx-native
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        with:
+          driver: remote
+          endpoint: tcp://localhost:1234
+          platforms: linux/amd64
+          append: |
+            - endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865
+              platforms: linux/arm64
+              name: mastodon-docker-builder-arm64-01
+              driver-opts:
+                - servername=mastodon-docker-builder-arm64-01
+        env:
+          BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }}
+          BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }}
+          BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }}
+
+      - name: Log in to Docker Hub
+        if: contains(inputs.push_to_images, 'tootsuite')
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Log in to the Github Container registry
+        if: contains(inputs.push_to_images, 'ghcr.io')
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/metadata-action@v4
+        id: meta
+        if: ${{ inputs.push_to_images != '' }}
+        with:
+          images: ${{ inputs.push_to_images }}
+          flavor: ${{ inputs.flavor }}
+          tags: ${{ inputs.tags }}
+          labels: ${{ inputs.labels }}
+
+      - uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: ${{ inputs.platforms }}
+          provenance: false
+          builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}
+          push: ${{ inputs.push_to_images != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: ${{ inputs.cache && 'type=gha' || '' }}
+          cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }}

.github/workflows/build-image.yml

@@ -1,70 +0,0 @@
-name: Build container image
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'main'
-    tags:
-      - '*'
-  pull_request:
-    paths:
-      - .github/workflows/build-image.yml
-      - Dockerfile
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-  build-image:
-    runs-on: ubuntu-latest
-
-    concurrency:
-      group: ${{ github.ref }}
-      cancel-in-progress: true
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: hadolint/hadolint-action@v3.1.0
-      - uses: docker/setup-qemu-action@v2
-      - uses: docker/setup-buildx-action@v2
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
-
-      - name: Log in to the Github Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
-
-      - uses: docker/metadata-action@v4
-        id: meta
-        with:
-          images: |
-            tootsuite/mastodon
-            ghcr.io/mastodon/mastodon
-          flavor: |
-            latest=auto
-          tags: |
-            type=edge,branch=main
-            type=pep440,pattern={{raw}}
-            type=pep440,pattern=v{{major}}.{{minor}}
-            type=ref,event=pr
-
-      - uses: docker/build-push-action@v4
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          provenance: false
-          builder: ${{ steps.buildx.outputs.name }}
-          push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/build-releases.yml

@@ -0,0 +1,27 @@
+name: Build container release images
+on:
+  push:
+    tags:
+      - '*'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-image:
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: true
+      push_to_images: |
+        tootsuite/mastodon
+        ghcr.io/mastodon/mastodon
+      # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
+      cache: false
+      flavor: |
+        latest=false
+      tags: |
+        type=pep440,pattern={{raw}}
+        type=pep440,pattern=v{{major}}.{{minor}}
+    secrets: inherit

.github/workflows/lint-ruby.yml

@@ -1,41 +0,0 @@
-name: Ruby Linting
-on:
-  push:
-    branches-ignore:
-      - 'dependabot/**'
-    paths:
-      - 'Gemfile*'
-      - '.rubocop.yml'
-      - '**/*.rb'
-      - '**/*.rake'
-      - '.github/workflows/lint-ruby.yml'
-
-  pull_request:
-    paths:
-      - 'Gemfile*'
-      - '.rubocop.yml'
-      - '**/*.rb'
-      - '**/*.rake'
-      - '.github/workflows/lint-ruby.yml'
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set-up RuboCop Problem Mathcher
-        uses: r7kamura/rubocop-problem-matchers-action@v1
-
-      - name: Run rubocop
-        uses: github/super-linter@v4
-        env:
-          DEFAULT_BRANCH: main
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          LINTER_RULES_PATH: .
-          RUBY_CONFIG_FILE: .rubocop.yml
-          VALIDATE_ALL_CODEBASE: false
-          VALIDATE_RUBY: true

.github/workflows/test-image-build.yml

@@ -0,0 +1,15 @@
+name: Test container image build
+on:
+  pull_request:
+permissions:
+  contents: read
+
+jobs:
+  build-image:
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      platforms: linux/amd64 # Testing only on native platform so it is performant

.ruby-version

@@ -1 +1 @@
-3.0.4
+3.0.6

CHANGELOG.md

@@ -3,6 +3,224 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.15] - 2024-02-16
+
+### Fixed
+
+- Fix OmniAuth tests and edge cases in error handling ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29201), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29207))
+
+### Security
+
+- Fix insufficient checking of remote posts ([GHSA-jhrq-qvrm-qr36](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36))
+
+## [4.1.14] - 2024-02-14
+
+### Security
+
+- Update the `sidekiq-unique-jobs` dependency (see [GHSA-cmh9-rx85-xj38](https://github.com/mhenrixon/sidekiq-unique-jobs/security/advisories/GHSA-cmh9-rx85-xj38))
+  In addition, we have disabled the web interface for `sidekiq-unique-jobs` out of caution.
+  If you need it, you can re-enable it by setting `ENABLE_SIDEKIQ_UNIQUE_JOBS_UI=true`.
+  If you only need to clear all locks, you can now use `bundle exec rake sidekiq_unique_jobs:delete_all_locks`.
+- Update the `nokogiri` dependency (see [GHSA-xc9x-jj77-9p9j](https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j))
+- Disable administrative Doorkeeper routes ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29187))
+- Fix ongoing streaming sessions not being invalidated when applications get deleted in some cases ([GHSA-7w3c-p9j8-mq3x](https://github.com/mastodon/mastodon/security/advisories/GHSA-7w3c-p9j8-mq3x))
+  In some rare cases, the streaming server was not notified of access tokens revocation on application deletion.
+- Change external authentication behavior to never reattach a new identity to an existing user by default ([GHSA-vm39-j3vx-pch3](https://github.com/mastodon/mastodon/security/advisories/GHSA-vm39-j3vx-pch3))
+  Up until now, Mastodon has allowed new identities from external authentication providers to attach to an existing local user based on their verified e-mail address.
+  This allowed upgrading users from a database-stored password to an external authentication provider, or move from one authentication provider to another.
+  However, this behavior may be unexpected, and means that when multiple authentication providers are configured, the overall security would be that of the least secure authentication provider.
+  For these reasons, this behavior is now locked under the `ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH` environment variable.
+  In addition, regardless of this environment variable, Mastodon will refuse to attach two identities from the same authentication provider to the same account.
+
+## [4.1.13] - 2024-02-01
+
+### Security
+
+- Fix insufficient origin validation (CVE-2024-23832, [GHSA-3fjr-858r-92rw](https://github.com/mastodon/mastodon/security/advisories/GHSA-3fjr-858r-92rw))
+
+## [4.1.12] - 2024-01-24
+
+### Fixed
+
+- Fix error when processing remote files with unusually long names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28823))
+- Fix processing of compacted single-item JSON-LD collections ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28816))
+- Retry 401 errors on replies fetching ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/28788))
+- Fix `RecordNotUnique` errors in LinkCrawlWorker ([tribela](https://github.com/mastodon/mastodon/pull/28748))
+- Fix Mastodon not correctly processing HTTP Signatures with query strings ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28443), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28476))
+- Fix potential redirection loop of streaming endpoint ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28665))
+- Fix streaming API redirection ignoring the port of `streaming_api_base_url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28558))
+- Fix `Undo Announce` activity not being sent to non-follower authors ([MitarashiDango](https://github.com/mastodon/mastodon/pull/18482))
+- Fix `LinkCrawlWorker` error when encountering empty OEmbed response ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28268))
+
+### Security
+
+- Add rate-limit of TOTP authentication attempts at controller level ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28801))
+
+## [4.1.11] - 2023-12-04
+
+### Changed
+
+- Change GIF max matrix size error to explicitly mention GIF files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27927))
+- Change `Follow` activities delivery to bypass availability check ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/27586))
+- Change Content-Security-Policy to be tighter on media paths ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26889))
+
+### Fixed
+
+- Fix incoming status creation date not being restricted to standard ISO8601 ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27655), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28081))
+- Fix posts from force-sensitized accounts being able to trend ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27620))
+- Fix processing LDSigned activities from actors with unknown public keys ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27474))
+- Fix error and incorrect URLs in `/api/v1/accounts/:id/featured_tags` for remote accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27459))
+- Fix report processing notice not mentioning the report number when performing a custom action ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27442))
+- Fix some link anchors being recognized as hashtags ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27271), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27584))
+
+## [4.1.10] - 2023-10-10
+
+### Changed
+
+- Change some worker lock TTLs to be shorter-lived ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27246))
+- Change user archive export allowed period from 7 days to 6 days ([suddjian](https://github.com/mastodon/mastodon/pull/27200))
+
+### Fixed
+
+- Fix mentions being matched in some URL query strings ([mjankowski](https://github.com/mastodon/mastodon/pull/25656))
+- Fix multiple instances of the trend refresh scheduler sometimes running at once ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27253))
+- Fix importer returning negative row estimates ([jgillich](https://github.com/mastodon/mastodon/pull/27258))
+- Fix filtering audit log for entries about disabling 2FA ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27186))
+- Fix tIME chunk not being properly removed from PNG uploads ([TheEssem](https://github.com/mastodon/mastodon/pull/27111))
+- Fix inefficient queries in “Follows and followers” as well as several admin pages ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27116), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27306))
+
+## [4.1.9] - 2023-09-20
+
+### Fixed
+
+- Fix post translation erroring out ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26990))
+
+## [4.1.8] - 2023-09-19
+
+### Fixed
+
+- Fix post edits not being forwarded as expected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26936))
+- Fix moderator rights inconsistencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26729))
+- Fix crash when encountering invalid URL ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26814))
+- Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409))
+- Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500))
+- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608))
+
+### Security
+
+- Fix missing HTML sanitization in translation API (CVE-2023-42452)
+- Fix incorrect domain name normalization (CVE-2023-42451)
+
+## [4.1.7] - 2023-09-05
+
+### Changed
+
+- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028))
+
+### Fixed
+
+- **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392))
+- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237))
+- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727))
+
+## [4.1.6] - 2023-07-31
+
+### Fixed
+
+- Fix memory leak in streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26228))
+- Fix wrong filters sometimes applying in streaming ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26159), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26213), [renchap](https://github.com/mastodon/mastodon/pull/26233))
+- Fix incorrect connect timeout in outgoing requests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26116))
+
+## [4.1.5] - 2023-07-21
+
+### Added
+
+- Add check preventing Sidekiq workers from running with Makara configured ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25850))
+
+### Changed
+
+- Change request timeout handling to use a longer deadline ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26055))
+
+### Fixed
+
+- Fix moderation interface for remote instances with a .zip TLD ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
+- Fix remote accounts being possibly persisted to database with incomplete protocol values ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
+- Fix trending publishers table not rendering correctly on narrow screens ([vmstan](https://github.com/mastodon/mastodon/pull/25945))
+
+### Security
+
+- Fix CSP headers being unintentionally wide ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26105))
+
+## [4.1.4] - 2023-07-07
+
+### Fixed
+
+- Fix branding:generate_app_icons failing because of disallowed ICO coder ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25794))
+- Fix crash in admin interface when viewing a remote user with verified links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25796))
+- Fix processing of media files with unusual names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25788))
+
+## [4.1.3] - 2023-07-06
+
+### Added
+
+- Add fallback redirection when getting a webfinger query `LOCAL_DOMAIN@LOCAL_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23600))
+
+### Changed
+
+- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058))
+- Change AccessTokensVacuum to also delete expired tokens ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24868))
+- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852))
+- Change automatic post deletion thresholds and load detection ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24614))
+- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510))
+- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216))
+
+### Removed
+
+- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070))
+
+### Fixed
+
+- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464))
+- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519))
+- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477))
+- Fix multiple inefficiencies in automatic post cleanup worker ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24785), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24840))
+- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361))
+- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273))
+- Fix `tootctl accounts approve --number N` not approving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605))
+- Fix reports not being closed when performing batch suspensions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24988))
+- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015))
+- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016))
+- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060))
+- Fix “Authorized applications” crashing when listing apps with certain admin API scopes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25713))
+- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499))
+- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431))
+- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637))
+- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342))
+
+### Security
+
+- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463))
+- Update dependencies
+- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756))
+- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462)
+- Fix timeout handling of outbound HTTP requests (CVE-2023-36461)
+- Fix arbitrary file creation through media processing (CVE-2023-36460)
+- Fix possible XSS in preview cards (CVE-2023-36459)
+
+## [4.1.2] - 2023-04-04
+
+### Fixed
+
+- Fix crash in `tootctl` commands making use of parallelization when Elasticsearch is enabled ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24182), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24377))
+- Fix crash in `db:setup` when Elasticsearch is enabled ([rrgeorge](https://github.com/mastodon/mastodon/pull/24302))
+- Fix user archive takeout when using OpenStack Swift or S3 providers with no ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24200))
+- Fix invalid/expired invites being processed on sign-up ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24337))
+
+### Security
+
+- Update Ruby to 3.0.6 due to ReDoS vulnerabilities ([saizai](https://github.com/mastodon/mastodon/pull/24334))
+- Fix unescaped user input in LDAP query ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24379))
+
 ## [4.1.1] - 2023-03-16
 
 ### Added

Dockerfile

@@ -2,7 +2,7 @@
 # This needs to be bullseye-slim because the Ruby image is built on bullseye-slim
 ARG NODE_VERSION="16.18.1-bullseye-slim"
 
-FROM ghcr.io/moritzheiber/ruby-jemalloc:3.0.4-slim as ruby
+FROM ghcr.io/moritzheiber/ruby-jemalloc:3.0.6-slim as ruby
 FROM node:${NODE_VERSION} as build
 
 COPY --link --from=ruby /opt/ruby /opt/ruby
@@ -17,6 +17,7 @@ COPY Gemfile* package.json yarn.lock /opt/mastodon/
 
 # hadolint ignore=DL3008
 RUN apt-get update && \
+    apt-get -yq dist-upgrade && \
     apt-get install -y --no-install-recommends build-essential \
         ca-certificates \
         git \

Gemfile.lock

@@ -10,40 +10,40 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actioncable (6.1.7.6)
+      actionpack (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activestorage (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionmailbox (6.1.7.6)
+      actionpack (= 6.1.7.6)
+      activejob (= 6.1.7.6)
+      activerecord (= 6.1.7.6)
+      activestorage (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       mail (>= 2.7.1)
-    actionmailer (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      actionview (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionmailer (6.1.7.6)
+      actionpack (= 6.1.7.6)
+      actionview (= 6.1.7.6)
+      activejob (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7.2)
-      actionview (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionpack (6.1.7.6)
+      actionview (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activestorage (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actiontext (6.1.7.6)
+      actionpack (= 6.1.7.6)
+      activerecord (= 6.1.7.6)
+      activestorage (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionview (6.1.7.6)
+      activesupport (= 6.1.7.6)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -54,22 +54,22 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.8)
-    activejob (6.1.7.2)
-      activesupport (= 6.1.7.2)
+    activejob (6.1.7.6)
+      activesupport (= 6.1.7.6)
       globalid (>= 0.3.6)
-    activemodel (6.1.7.2)
-      activesupport (= 6.1.7.2)
-    activerecord (6.1.7.2)
-      activemodel (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
-    activestorage (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    activemodel (6.1.7.6)
+      activesupport (= 6.1.7.6)
+    activerecord (6.1.7.6)
+      activemodel (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
+    activestorage (6.1.7.6)
+      actionpack (= 6.1.7.6)
+      activejob (= 6.1.7.6)
+      activerecord (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7.2)
+    activesupport (6.1.7.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -120,8 +120,7 @@ GEM
     bindata (2.4.14)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
-    blurhash (0.1.6)
-      ffi (~> 1.14)
+    blurhash (0.1.7)
     bootsnap (1.16.0)
       msgpack (~> 1.2)
     brakeman (5.4.0)
@@ -174,7 +173,7 @@ GEM
     cocoon (1.2.15)
     coderay (1.1.3)
     color_diff (0.1)
-    concurrent-ruby (1.2.0)
+    concurrent-ruby (1.2.2)
     connection_pool (2.3.0)
     cose (1.2.1)
       cbor (~> 0.5.9)
@@ -207,7 +206,7 @@ GEM
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    doorkeeper (5.6.4)
+    doorkeeper (5.6.6)
       railties (>= 5)
     dotenv (2.8.1)
     dotenv-rails (2.8.1)
@@ -389,7 +388,7 @@ GEM
     loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    mail (2.8.0.1)
+    mail (2.8.1)
       mini_mime (>= 0.1.1)
       net-imap
       net-pop
@@ -405,13 +404,13 @@ GEM
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
-    mini_mime (1.1.2)
-    mini_portile2 (2.8.1)
+    mini_mime (1.1.5)
+    mini_portile2 (2.8.5)
     minitest (5.17.0)
     msgpack (1.6.0)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    net-imap (0.3.4)
+    net-imap (0.3.7)
       date
       net-protocol
     net-ldap (0.17.1)
@@ -424,9 +423,9 @@ GEM
     net-smtp (0.3.3)
       net-protocol
     net-ssh (7.0.1)
-    nio4r (2.5.8)
-    nokogiri (1.14.1)
-      mini_portile2 (~> 2.8.0)
+    nio4r (2.5.9)
+    nokogiri (1.16.2)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.2.8)
       activesupport (>= 4.2, < 7)
@@ -469,7 +468,7 @@ GEM
     parslet (2.0.0)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pg (1.4.5)
+    pg (1.4.6)
     pghero (3.1.0)
       activerecord (>= 6)
     pkg-config (1.5.1)
@@ -492,13 +491,13 @@ GEM
     pry-rails (0.3.9)
       pry (>= 0.10.4)
     public_suffix (5.0.1)
-    puma (5.6.5)
+    puma (5.6.7)
       nio4r (~> 2.0)
     pundit (2.3.0)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
-    racc (1.6.2)
-    rack (2.2.6.2)
+    racc (1.7.3)
+    rack (2.2.8)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
@@ -513,20 +512,20 @@ GEM
       rack
     rack-test (2.0.2)
       rack (>= 1.3)
-    rails (6.1.7.2)
-      actioncable (= 6.1.7.2)
-      actionmailbox (= 6.1.7.2)
-      actionmailer (= 6.1.7.2)
-      actionpack (= 6.1.7.2)
-      actiontext (= 6.1.7.2)
-      actionview (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activemodel (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activestorage (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    rails (6.1.7.6)
+      actioncable (= 6.1.7.6)
+      actionmailbox (= 6.1.7.6)
+      actionmailer (= 6.1.7.6)
+      actionpack (= 6.1.7.6)
+      actiontext (= 6.1.7.6)
+      actionview (= 6.1.7.6)
+      activejob (= 6.1.7.6)
+      activemodel (= 6.1.7.6)
+      activerecord (= 6.1.7.6)
+      activestorage (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       bundler (>= 1.15.0)
-      railties (= 6.1.7.2)
+      railties (= 6.1.7.6)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -542,9 +541,9 @@ GEM
       railties (>= 6.0.0, < 7)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    railties (6.1.7.6)
+      actionpack (= 6.1.7.6)
+      activesupport (= 6.1.7.6)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -628,14 +627,14 @@ GEM
       fugit (~> 1.1, >= 1.1.6)
     safety_net_attestation (0.4.0)
       jwt (~> 2.0)
-    sanitize (6.0.1)
+    sanitize (6.0.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     scenic (1.7.0)
       activerecord (>= 4.0.0)
       railties (>= 4.0.0)
     semantic_range (3.0.0)
-    sidekiq (6.5.8)
+    sidekiq (6.5.12)
       connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
       redis (>= 4.5.0, < 5)
@@ -646,7 +645,7 @@ GEM
       rufus-scheduler (~> 3.2)
       sidekiq (>= 4, < 7)
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (7.1.29)
+    sidekiq-unique-jobs (7.1.33)
       brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.5)
       redis (< 5.0)
@@ -689,9 +688,9 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
-    thor (1.2.1)
+    thor (1.2.2)
     tilt (2.0.11)
-    timeout (0.3.1)
+    timeout (0.3.2)
     tpm-key_attestation (0.11.0)
       bindata (~> 2.4)
       openssl (> 2.0, < 3.1)
@@ -747,14 +746,14 @@ GEM
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     wisper (2.0.1)
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.6)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   ruby

SECURITY.md

@@ -10,8 +10,8 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 ## Supported Versions
 
-| Version | Supported |
-| ------- | ----------|
-| 4.0.x   | Yes       |
-| 3.5.x   | Yes       |
-| < 3.5   | No        |
+| Version | Supported        |
+| ------- | ---------------- |
+| 4.2.x   | Yes              |
+| 4.1.x   | Yes              |
+| < 4.1   | No               |

app/chewy/accounts_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class AccountsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: { refresh_interval: '30s' }, analysis: {
     analyzer: {
       content: {
@@ -38,6 +40,6 @@ class AccountsIndex < Chewy::Index
 
     field :following_count, type: 'long', value: ->(account) { account.following_count }
     field :followers_count, type: 'long', value: ->(account) { account.followers_count }
-    field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    field :last_status_at, type: 'date', value: ->(account) { clamp_date(account.last_status_at || account.created_at) }
   end
 end

app/chewy/concerns/datetime_clamping_concern.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DatetimeClampingConcern
+  extend ActiveSupport::Concern
+
+  MIN_ISO8601_DATETIME = '0000-01-01T00:00:00Z'.to_datetime.freeze
+  MAX_ISO8601_DATETIME = '9999-12-31T23:59:59Z'.to_datetime.freeze
+
+  class_methods do
+    def clamp_date(datetime)
+      datetime.clamp(MIN_ISO8601_DATETIME, MAX_ISO8601_DATETIME)
+    end
+  end
+end

app/chewy/tags_index.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class TagsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: { refresh_interval: '30s' }, analysis: {
     analyzer: {
       content: {
@@ -36,6 +38,6 @@ class TagsIndex < Chewy::Index
 
     field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
     field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts }
-    field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    field :last_status_at, type: 'date', value: ->(tag) { clamp_date(tag.last_status_at || tag.created_at) }
   end
 end

app/controllers/admin/account_actions_controller.rb

@@ -21,7 +21,7 @@ module Admin
       account_action.save!
 
       if account_action.with_report?
-        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: params[:report_id])
+        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: resource_params[:report_id])
       else
         redirect_to admin_account_path(@account.id)
       end

app/controllers/admin/domain_blocks_controller.rb

@@ -37,7 +37,7 @@ module Admin
         @domain_block.errors.delete(:domain)
         render :new
       else
-        if existing_domain_block.present?
+        if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip)
           @domain_block = existing_domain_block
           @domain_block.update(resource_params)
         end

app/controllers/admin/webhooks_controller.rb

@@ -20,6 +20,7 @@ module Admin
       authorize :webhook, :create?
 
       @webhook = Webhook.new(resource_params)
+      @webhook.current_account = current_account
 
       if @webhook.save
         redirect_to admin_webhook_path(@webhook)
@@ -39,10 +40,12 @@ module Admin
     def update
       authorize @webhook, :update?
 
+      @webhook.current_account = current_account
+
       if @webhook.update(resource_params)
         redirect_to admin_webhook_path(@webhook)
       else
-        render :show
+        render :edit
       end
     end
 

app/controllers/api/v1/conversations_controller.rb

@@ -11,7 +11,7 @@ class Api::V1::ConversationsController < Api::BaseController
 
   def index
     @conversations = paginated_conversations
-    render json: @conversations, each_serializer: REST::ConversationSerializer
+    render json: @conversations, each_serializer: REST::ConversationSerializer, relationships: StatusRelationshipsPresenter.new(@conversations.map(&:last_status), current_user&.account_id)
   end
 
   def read
@@ -32,6 +32,19 @@ class Api::V1::ConversationsController < Api::BaseController
 
   def paginated_conversations
     AccountConversation.where(account: current_account)
+                       .includes(
+                         account: :account_stat,
+                         last_status: [
+                           :media_attachments,
+                           :preview_cards,
+                           :status_stat,
+                           :tags,
+                           {
+                             active_mentions: [account: :account_stat],
+                             account: :account_stat,
+                           },
+                         ]
+                       )
                        .to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end
 

app/controllers/api/v1/statuses/histories_controller.rb

@@ -7,11 +7,15 @@ class Api::V1::Statuses::HistoriesController < Api::BaseController
   before_action :set_status
 
   def show
-    render json: @status.edits.includes(:account, status: [:account]), each_serializer: REST::StatusEditSerializer
+    render json: status_edits, each_serializer: REST::StatusEditSerializer
   end
 
   private
 
+  def status_edits
+    @status.edits.includes(:account, status: [:account]).to_a.presence || [@status.build_snapshot(at_time: @status.edited_at || @status.created_at)]
+  end
+
   def set_status
     @status = Status.find(params[:status_id])
     authorize @status, :show?

app/controllers/api/v1/statuses/reblogs_controller.rb

@@ -2,6 +2,8 @@
 
 class Api::V1::Statuses::ReblogsController < Api::BaseController
   include Authorization
+  include Redisable
+  include Lockable
 
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }
   before_action :require_user!
@@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   override_rate_limit_headers :create, family: :statuses
 
   def create
-    @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    with_lock("reblog:#{current_account.id}:#{@reblog.id}") do
+      @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    end
 
     render json: @status, serializer: REST::StatusSerializer
   end

app/controllers/api/v1/streaming_controller.rb

@@ -2,7 +2,7 @@
 
 class Api::V1::StreamingController < Api::BaseController
   def index
-    if Rails.configuration.x.streaming_api_base_url == request.host
+    if same_host?
       not_found
     else
       redirect_to streaming_api_url, status: 301
@@ -11,9 +11,16 @@ class Api::V1::StreamingController < Api::BaseController
 
   private
 
+  def same_host?
+    base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
+    request.host == base_url.host && request.port == (base_url.port || 80)
+  end
+
   def streaming_api_url
     Addressable::URI.parse(request.url).tap do |uri|
-      uri.host = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url).host
+      base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
+      uri.host = base_url.host
+      uri.port = base_url.port
     end.to_s
   end
 end

app/controllers/api/v1/timelines/tag_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::TagController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth?
   before_action :load_tag
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
@@ -11,6 +12,10 @@ class Api::V1::Timelines::TagController < Api::BaseController
 
   private
 
+  def require_auth?
+    !Setting.timeline_preview
+  end
+
   def load_tag
     @tag = Tag.find_normalized(params[:id])
   end

app/controllers/api/v2/admin/accounts_controller.rb

@@ -18,6 +18,14 @@ class Api::V2::Admin::AccountsController < Api::V1::Admin::AccountsController
 
   private
 
+  def next_path
+    api_v2_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v2_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
+  end
+
   def filtered_accounts
     AccountFilter.new(translated_filter_params).results
   end

app/controllers/auth/omniauth_callbacks_controller.rb

@@ -5,7 +5,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
 
   def self.provides_callback_for(provider)
     define_method provider do
-      @user = User.find_for_oauth(request.env['omniauth.auth'], current_user)
+      @user = User.find_for_omniauth(request.env['omniauth.auth'], current_user)
 
       if @user.persisted?
         LoginActivity.create(
@@ -24,6 +24,9 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
         session["devise.#{provider}_data"] = request.env['omniauth.auth']
         redirect_to new_user_registration_url
       end
+    rescue ActiveRecord::RecordInvalid
+      flash[:alert] = I18n.t('devise.failure.omniauth_user_creation_failure') if is_navigational_format?
+      redirect_to new_user_session_url
     end
   end
 

app/controllers/auth/registrations_controller.rb

@@ -48,7 +48,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
     super(hash)
 
     resource.locale                 = I18n.locale
-    resource.invite_code            = params[:invite_code] if resource.invite_code.blank?
+    resource.invite_code            = @invite&.code if resource.invite_code.blank?
     resource.registration_form_time = session[:registration_form_time]
     resource.sign_up_ip             = request.remote_ip
 

app/controllers/auth/sessions_controller.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class Auth::SessionsController < Devise::SessionsController
+  include Redisable
+
+  MAX_2FA_ATTEMPTS_PER_HOUR = 10
+
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
@@ -136,9 +140,23 @@ class Auth::SessionsController < Devise::SessionsController
     session.delete(:attempt_user_updated_at)
   end
 
+  def clear_2fa_attempt_from_user(user)
+    redis.del(second_factor_attempts_key(user))
+  end
+
+  def check_second_factor_rate_limits(user)
+    attempts, = redis.multi do |multi|
+      multi.incr(second_factor_attempts_key(user))
+      multi.expire(second_factor_attempts_key(user), 1.hour)
+    end
+
+    attempts >= MAX_2FA_ATTEMPTS_PER_HOUR
+  end
+
   def on_authentication_success(user, security_measure)
     @on_authentication_success_called = true
 
+    clear_2fa_attempt_from_user(user)
     clear_attempt_from_session
 
     user.update_sign_in!(new_sign_in: true)
@@ -170,4 +188,8 @@ class Auth::SessionsController < Devise::SessionsController
       user_agent: request.user_agent
     )
   end
+
+  def second_factor_attempts_key(user)
+    "2fa_auth_attempts:#{user.id}:#{Time.now.utc.hour}"
+  end
 end

app/controllers/backups_controller.rb

@@ -13,7 +13,11 @@ class BackupsController < ApplicationController
     when :s3
       redirect_to @backup.dump.expiring_url(10)
     when :fog
-      redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
+      if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present?
+        redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
+      else
+        redirect_to full_asset_url(@backup.dump.url)
+      end
     when :filesystem
       redirect_to full_asset_url(@backup.dump.url)
     end

app/controllers/concerns/signature_verification.rb

@@ -91,14 +91,23 @@ module SignatureVerification
     raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if actor.nil?
 
     signature             = Base64.decode64(signature_params['signature'])
-    compare_signed_string = build_signed_string
+    compare_signed_string = build_signed_string(include_query_string: true)
 
     return actor unless verify_signature(actor, signature, compare_signed_string).nil?
 
+    # Compatibility quirk with older Mastodon versions
+    compare_signed_string = build_signed_string(include_query_string: false)
+    return actor unless verify_signature(actor, signature, compare_signed_string).nil?
+
     actor = stoplight_wrap_request { actor_refresh_key!(actor) }
 
     raise SignatureVerificationError, "Could not refresh public key #{signature_params['keyId']}" if actor.nil?
 
+    compare_signed_string = build_signed_string(include_query_string: true)
+    return actor unless verify_signature(actor, signature, compare_signed_string).nil?
+
+    # Compatibility quirk with older Mastodon versions
+    compare_signed_string = build_signed_string(include_query_string: false)
     return actor unless verify_signature(actor, signature, compare_signed_string).nil?
 
     fail_with! "Verification failed for #{actor.to_log_human_identifier} #{actor.uri} using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)", signed_string: compare_signed_string, signature: signature_params['signature']
@@ -177,16 +186,24 @@ module SignatureVerification
     nil
   end
 
-  def build_signed_string
+  def build_signed_string(include_query_string: true)
     signed_headers.map do |signed_header|
-      if signed_header == Request::REQUEST_TARGET
-        "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
-      elsif signed_header == '(created)'
+      case signed_header
+      when Request::REQUEST_TARGET
+        if include_query_string
+          "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.original_fullpath}"
+        else
+          # Current versions of Mastodon incorrectly omit the query string from the (request-target) pseudo-header.
+          # Therefore, temporarily support such incorrect signatures for compatibility.
+          # TODO: remove eventually some time after release of the fixed version
+          "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
+        end
+      when '(created)'
         raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank?
 
         "(created): #{signature_params['created']}"
-      elsif signed_header == '(expires)'
+      when '(expires)'
         raise SignatureVerificationError, 'Invalid pseudo-header (expires) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (expires) used but corresponding argument missing' if signature_params['expires'].blank?
 
@@ -246,7 +263,7 @@ module SignatureVerification
       stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, ''), suppress_errors: false) }
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
       account   = ActivityPub::TagManager.instance.uri_to_actor(key_id)
-      account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false, suppress_errors: false) }
+      account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, suppress_errors: false) }
       account
     end
   rescue Mastodon::PrivateNetworkAddressError => e

app/controllers/concerns/two_factor_authentication_concern.rb

@@ -65,6 +65,11 @@ module TwoFactorAuthenticationConcern
   end
 
   def authenticate_with_two_factor_via_otp(user)
+    if check_second_factor_rate_limits(user)
+      flash.now[:alert] = I18n.t('users.rate_limited')
+      return prompt_for_two_factor(user)
+    end
+
     if valid_otp_attempt?(user)
       on_authentication_success(user, :otp)
     else

app/controllers/media_controller.rb

@@ -46,6 +46,6 @@ class MediaController < ApplicationController
   end
 
   def allow_iframing
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
+    response.headers.delete('X-Frame-Options')
   end
 end

app/controllers/oauth/authorized_applications_controller.rb

@@ -8,6 +8,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   before_action :require_not_suspended!, only: :destroy
   before_action :set_body_classes
 
+  before_action :set_last_used_at_by_app, only: :index, unless: -> { request.format == :json }
+
   skip_before_action :require_functional!
 
   include Localized
@@ -30,4 +32,14 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   def require_not_suspended!
     forbidden if current_account.suspended?
   end
+
+  def set_last_used_at_by_app
+    @last_used_at_by_app = Doorkeeper::AccessToken
+                           .select('DISTINCT ON (application_id) application_id, last_used_at')
+                           .where(resource_owner_id: current_resource_owner.id)
+                           .where.not(last_used_at: nil)
+                           .order(application_id: :desc, last_used_at: :desc)
+                           .pluck(:application_id, :last_used_at)
+                           .to_h
+  end
 end

app/controllers/statuses_controller.rb

@@ -43,7 +43,7 @@ class StatusesController < ApplicationController
     return not_found if @status.hidden? || @status.reblog?
 
     expires_in 180, public: true
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
+    response.headers.delete('X-Frame-Options')
 
     render layout: 'embedded'
   end

app/controllers/well_known/webfinger_controller.rb

@@ -18,7 +18,14 @@ module WellKnown
     private
 
     def set_account
-      @account = Account.find_local!(username_from_resource)
+      username = username_from_resource
+      @account = begin
+        if username == Rails.configuration.x.local_domain
+          Account.representative
+        else
+          Account.find_local!(username)
+        end
+      end
     end
 
     def username_from_resource

app/helpers/formatting_helper.rb

@@ -58,6 +58,10 @@ module FormattingHelper
   end
 
   def account_field_value_format(field, with_rel_me: true)
-    html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
+    if field.verified? && !field.account.local?
+      TextFormatter.shortened_link(field.value_for_verification)
+    else
+      html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
+    end
   end
 end

app/helpers/jsonld_helper.rb

@@ -157,8 +157,8 @@ module JsonLdHelper
     end
   end
 
-  def fetch_resource(uri, id, on_behalf_of = nil)
-    unless id
+  def fetch_resource(uri, id_is_known, on_behalf_of = nil, request_options: {})
+    unless id_is_known
       json = fetch_resource_without_id_validation(uri, on_behalf_of)
 
       return if !json.is_a?(Hash) || unsupported_uri_scheme?(json['id'])
@@ -166,17 +166,29 @@ module JsonLdHelper
       uri = json['id']
     end
 
-    json = fetch_resource_without_id_validation(uri, on_behalf_of)
+    json = fetch_resource_without_id_validation(uri, on_behalf_of, request_options: request_options)
     json.present? && json['id'] == uri ? json : nil
   end
 
-  def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
+  def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false, request_options: {})
     on_behalf_of ||= Account.representative
 
-    build_request(uri, on_behalf_of).perform do |response|
+    build_request(uri, on_behalf_of, options: request_options).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
-      body_to_json(response.body_with_limit) if response.code == 200
+      body_to_json(response.body_with_limit) if response.code == 200 && valid_activitypub_content_type?(response)
+    end
+  end
+
+  def valid_activitypub_content_type?(response)
+    return true if response.mime_type == 'application/activity+json'
+
+    # When the mime type is `application/ld+json`, we need to check the profile,
+    # but `http.rb` does not parse it for us.
+    return false unless response.mime_type == 'application/ld+json'
+
+    response.headers[HTTP::Headers::CONTENT_TYPE]&.split(';')&.map(&:strip)&.any? do |str|
+      str.start_with?('profile="') && str[9...-1].split.include?('https://www.w3.org/ns/activitystreams')
     end
   end
 
@@ -206,8 +218,8 @@ module JsonLdHelper
     response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
   end
 
-  def build_request(uri, on_behalf_of = nil)
-    Request.new(:get, uri).tap do |request|
+  def build_request(uri, on_behalf_of = nil, options: {})
+    Request.new(:get, uri, **options).tap do |request|
       request.on_behalf_of(on_behalf_of) if on_behalf_of
       request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     end

app/lib/account_reach_finder.rb

@@ -6,7 +6,7 @@ class AccountReachFinder
   end
 
   def inboxes
-    (followers_inboxes + reporters_inboxes + relay_inboxes).uniq
+    (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq
   end
 
   private
@@ -19,6 +19,13 @@ class AccountReachFinder
     Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
   end
 
+  def recently_mentioned_inboxes
+    cutoff_id       = Mastodon::Snowflake.id_at(2.days.ago, with_random: false)
+    recent_statuses = @account.statuses.recent.where(id: cutoff_id...).limit(200)
+
+    Account.joins(:mentions).where(mentions: { status: recent_statuses }).inboxes.take(2000)
+  end
+
   def relay_inboxes
     Relay.enabled.pluck(:inbox_url)
   end

app/lib/activitypub/activity.rb

@@ -153,7 +153,8 @@ class ActivityPub::Activity
   def fetch_remote_original_status
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
+
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
     elsif @object['url'].present?
       ::FetchRemoteStatusService.new.call(@object['url'], request_id: @options[:request_id])
     end

app/lib/activitypub/activity/flag.rb

@@ -16,7 +16,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
         @account,
         target_account,
         status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id),
-        comment: @json['content'] || '',
+        comment: report_comment,
         uri: report_uri
       )
     end
@@ -35,4 +35,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
   def report_uri
     @json['id'] unless @json['id'].nil? || invalid_origin?(@json['id'])
   end
+
+  def report_comment
+    (@json['content'] || '')[0...5000]
+  end
 end

app/lib/activitypub/activity/update.rb

@@ -28,6 +28,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
 
     return if @status.nil?
 
-    ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id])
+    ActivityPub::ProcessStatusUpdateService.new.call(@status, @json, @object, request_id: @options[:request_id])
   end
 end

app/lib/activitypub/linked_data_signature.rb

@@ -18,8 +18,8 @@ class ActivityPub::LinkedDataSignature
 
     return unless type == 'RsaSignature2017'
 
-    creator   = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
-    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false)
+    creator = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
+    creator = ActivityPub::FetchRemoteKeyService.new.call(creator_uri) if creator&.public_key.blank?
 
     return if creator.nil?
 
@@ -27,9 +27,9 @@ class ActivityPub::LinkedDataSignature
     document_hash  = hash(@json.without('signature'))
     to_be_verified = options_hash + document_hash
 
-    if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
-      creator
-    end
+    creator if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
+  rescue OpenSSL::PKey::RSAError
+    false
   end
 
   def sign!(creator, sign_with: nil)

app/lib/activitypub/parser/status_parser.rb

@@ -53,7 +53,8 @@ class ActivityPub::Parser::StatusParser
   end
 
   def created_at
-    @object['published']&.to_datetime
+    datetime = @object['published']&.to_datetime
+    datetime if datetime.present? && (0..9999).cover?(datetime.year)
   rescue ArgumentError
     nil
   end

app/lib/activitypub/tag_manager.rb

@@ -27,6 +27,8 @@ class ActivityPub::TagManager
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
+    when :flag
+      target.uri
     end
   end
 
@@ -41,6 +43,8 @@ class ActivityPub::TagManager
       account_status_url(target.account, target)
     when :emoji
       emoji_url(target)
+    when :flag
+      target.uri
     end
   end
 

app/lib/admin/account_statuses_filter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Admin::AccountStatusesFilter < AccountStatusesFilter
+  private
+
+  def blocked?
+    false
+  end
+end

app/lib/application_extension.rb

@@ -4,16 +4,32 @@ module ApplicationExtension
   extend ActiveSupport::Concern
 
   included do
+    include Redisable
+
     validates :name, length: { maximum: 60 }
     validates :website, url: true, length: { maximum: 2_000 }, if: :website?
     validates :redirect_uri, length: { maximum: 2_000 }
-  end
 
-  def most_recently_used_access_token
-    @most_recently_used_access_token ||= access_tokens.where.not(last_used_at: nil).order(last_used_at: :desc).first
+    # The relationship used between Applications and AccessTokens is using
+    # dependent: delete_all, which means the ActiveRecord callback in
+    # AccessTokenExtension is not run, so instead we manually announce to
+    # streaming that these tokens are being deleted.
+    before_destroy :push_to_streaming_api, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
+
+  def push_to_streaming_api
+    # TODO: #28793 Combine into a single topic
+    payload = Oj.dump(event: :kill)
+    access_tokens.in_batches do |tokens|
+      redis.pipelined do |pipeline|
+        tokens.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
+    end
+  end
 end

app/lib/importer/base_importer.rb

@@ -34,7 +34,9 @@ class Importer::BaseImporter
   # Estimate the amount of documents that would be indexed. Not exact!
   # @returns [Integer]
   def estimate!
-    ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples AS estimate FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['estimate'].to_i }
+    reltuples = ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['reltuples'].to_i }
+    # If the table has never yet been vacuumed or analyzed, reltuples contains -1
+    [reltuples, 0].max
   end
 
   # Import data from the database into the index

app/lib/link_details_extractor.rb

@@ -140,7 +140,7 @@ class LinkDetailsExtractor
   end
 
   def html
-    player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
+    player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowfullscreen: 'true', allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
   end
 
   def width

app/lib/plain_text_formatter.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 class PlainTextFormatter
-  include ActionView::Helpers::TextHelper
-
-  NEWLINE_TAGS_RE = /(<br \/>|<br>|<\/p>)+/.freeze
+  NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
 
   attr_reader :text, :local
 
@@ -18,7 +16,10 @@ class PlainTextFormatter
     if local?
       text
     else
-      html_entities.decode(strip_tags(insert_newlines)).chomp
+      node = Nokogiri::HTML.fragment(insert_newlines)
+      # Elements that are entirely removed with our Sanitize config
+      node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
+      node.text.chomp
     end
   end
 
@@ -27,8 +28,4 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
-
-  def html_entities
-    HTMLEntities.new
-  end
 end

app/lib/request.rb

@@ -4,14 +4,60 @@ require 'ipaddr'
 require 'socket'
 require 'resolv'
 
-# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
+# Use our own timeout class to avoid using HTTP.rb's timeout block
 # around the Socket#open method, since we use our own timeout blocks inside
 # that method
-class HTTP::Timeout::PerOperation
+#
+# Also changes how the read timeout behaves so that it is cumulative (closer
+# to HTTP::Timeout::Global, but still having distinct timeouts for other
+# operation types)
+class PerOperationWithDeadline < HTTP::Timeout::PerOperation
+  READ_DEADLINE = 30
+
+  def initialize(*args)
+    super
+
+    @read_deadline = options.fetch(:read_deadline, READ_DEADLINE)
+  end
+
   def connect(socket_class, host, port, nodelay = false)
     @socket = socket_class.open(host, port)
     @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
   end
+
+  # Reset deadline when the connection is re-used for different requests
+  def reset_counter
+    @deadline = nil
+  end
+
+  # Read data from the socket
+  def readpartial(size, buffer = nil)
+    @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_deadline
+
+    timeout = false
+    loop do
+      result = @socket.read_nonblock(size, buffer, exception: false)
+
+      return :eof if result.nil?
+
+      remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout
+      raise HTTP::TimeoutError, "Read timed out after a total of #{@read_deadline} seconds" if remaining_time <= 0
+      return result if result != :wait_readable
+
+      # marking the socket for timeout. Why is this not being raised immediately?
+      # it seems there is some race-condition on the network level between calling
+      # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
+      # for reads, and when waiting for x seconds, it returns nil suddenly without completing
+      # the x seconds. In a normal case this would be a timeout on wait/read, but it can
+      # also mean that the socket has been closed by the server. Therefore we "mark" the
+      # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
+      # timeout. Else, the first timeout was a proper timeout.
+      # This hack has to be done because io/wait#wait_readable doesn't provide a value for when
+      # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
+      timeout = true unless @socket.to_io.wait_readable([remaining_time, @read_timeout].min)
+    end
+  end
 end
 
 class Request
@@ -20,7 +66,7 @@ class Request
   # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
   # and 5s timeout on the TLS handshake, meaning the worst case should take
   # about 15s in total
-  TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
+  TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze
 
   include RoutingHelper
 
@@ -31,7 +77,9 @@ class Request
     @url         = Addressable::URI.parse(url).normalize
     @http_client = options.delete(:http_client)
     @allow_local = options.delete(:allow_local)
+    @full_path   = options.delete(:with_query_string)
     @options     = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
+    @options     = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT)
     @options     = @options.merge(proxy_url) if use_proxy?
     @headers     = {}
 
@@ -92,14 +140,14 @@ class Request
     end
 
     def http_client
-      HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
+      HTTP.use(:auto_inflate).follow(max_hops: 3)
     end
   end
 
   private
 
   def set_common_headers!
-    @headers[REQUEST_TARGET]    = "#{@verb} #{@url.path}"
+    @headers[REQUEST_TARGET]    = request_target
     @headers['User-Agent']      = Mastodon::Version.user_agent
     @headers['Host']            = @url.host
     @headers['Date']            = Time.now.utc.httpdate
@@ -110,6 +158,14 @@ class Request
     @headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
   end
 
+  def request_target
+    if @url.query.nil? || !@full_path
+      "#{@verb} #{@url.path}"
+    else
+      "#{@verb} #{@url.path}?#{@url.query}"
+    end
+  end
+
   def signature
     algorithm = 'rsa-sha256'
     signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
@@ -238,11 +294,11 @@ class Request
         end
 
         until socks.empty?
-          _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
+          _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect_timeout])
 
           if available_socks.nil?
             socks.each(&:close)
-            raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
+            raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect_timeout]} seconds"
           end
 
           available_socks.each do |sock|

app/lib/scope_parser.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ScopeParser < Parslet::Parser
-  rule(:term)      { match('[a-z]').repeat(1).as(:term) }
+  rule(:term)      { match('[a-z_]').repeat(1).as(:term) }
   rule(:colon)     { str(':') }
   rule(:access)    { (str('write') | str('read')).as(:access) }
   rule(:namespace) { str('admin').as(:namespace) }

app/lib/status_reach_finder.rb

@@ -16,28 +16,28 @@ class StatusReachFinder
   private
 
   def reached_account_inboxes
+    Account.where(id: reached_account_ids).inboxes
+  end
+
+  def reached_account_ids
     # When the status is a reblog, there are no interactions with it
     # directly, we assume all interactions are with the original one
 
     if @status.reblog?
-      []
+      [reblog_of_account_id]
     else
-      Account.where(id: reached_account_ids).inboxes
-    end
-  end
-
-  def reached_account_ids
-    [
-      replied_to_account_id,
-      reblog_of_account_id,
-      mentioned_account_ids,
-      reblogs_account_ids,
-      favourites_account_ids,
-      replies_account_ids,
-    ].tap do |arr|
-      arr.flatten!
-      arr.compact!
-      arr.uniq!
+      [
+        replied_to_account_id,
+        reblog_of_account_id,
+        mentioned_account_ids,
+        reblogs_account_ids,
+        favourites_account_ids,
+        replies_account_ids,
+      ].tap do |arr|
+        arr.flatten!
+        arr.compact!
+        arr.uniq!
+      end
     end
   end
 

app/lib/tag_manager.rb

@@ -7,18 +7,18 @@ class TagManager
   include RoutingHelper
 
   def web_domain?(domain)
-    domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.web_domain).zero?
+    domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.web_domain).zero?
   end
 
   def local_domain?(domain)
-    domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero?
+    domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.local_domain).zero?
   end
 
   def normalize_domain(domain)
     return if domain.nil?
 
     uri = Addressable::URI.new
-    uri.host = domain.gsub(/[\/]/, '')
+    uri.host = domain.delete_suffix('/')
     uri.normalized_host
   end
 
@@ -28,7 +28,7 @@ class TagManager
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
 
     TagManager.instance.web_domain?(domain)
-  rescue Addressable::URI::InvalidURIError
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
     false
   end
 end

app/lib/text_formatter.rb

@@ -48,6 +48,26 @@ class TextFormatter
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
+  class << self
+    include ERB::Util
+
+    def shortened_link(url, rel_me: false)
+      url = Addressable::URI.parse(url).to_s
+      rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
+
+      prefix      = url.match(URL_PREFIX_REGEX).to_s
+      display_url = url[prefix.length, 30]
+      suffix      = url[prefix.length + 30..-1]
+      cutoff      = url[prefix.length..-1].length > 30
+
+      <<~HTML.squish.html_safe # rubocop:disable Rails/OutputSafety
+        <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
+      HTML
+    rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+      h(url)
+    end
+  end
+
   private
 
   def rewrite
@@ -70,19 +90,7 @@ class TextFormatter
   end
 
   def link_to_url(entity)
-    url = Addressable::URI.parse(entity[:url]).to_s
-    rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
-
-    prefix      = url.match(URL_PREFIX_REGEX).to_s
-    display_url = url[prefix.length, 30]
-    suffix      = url[prefix.length + 30..-1]
-    cutoff      = url[prefix.length..-1].length > 30
-
-    <<~HTML.squish
-      <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
-    HTML
-  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
-    h(entity[:url])
+    TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?)
   end
 
   def link_to_hashtag(entity)

app/lib/translation_service/deepl.rb

@@ -46,7 +46,7 @@ class TranslationService::DeepL < TranslationService
 
     raise UnexpectedResponseError unless json.is_a?(Hash)
 
-    Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
+    Translation.new(text: Sanitize.fragment(json.dig('translations', 0, 'text'), Sanitize::Config::MASTODON_STRICT), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
   rescue Oj::ParseError
     raise UnexpectedResponseError
   end

app/lib/translation_service/libre_translate.rb

@@ -37,7 +37,7 @@ class TranslationService::LibreTranslate < TranslationService
 
     raise UnexpectedResponseError unless json.is_a?(Hash)
 
-    Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate')
+    Translation.new(text: Sanitize.fragment(json['translatedText'], Sanitize::Config::MASTODON_STRICT), detected_source_language: source_language, provider: 'LibreTranslate')
   rescue Oj::ParseError
     raise UnexpectedResponseError
   end

app/lib/vacuum/access_tokens_vacuum.rb

@@ -9,10 +9,12 @@ class Vacuum::AccessTokensVacuum
   private
 
   def vacuum_revoked_access_tokens!
-    Doorkeeper::AccessToken.where.not(revoked_at: nil).where('revoked_at < NOW()').delete_all
+    Doorkeeper::AccessToken.where.not(expires_in: nil).where('created_at + make_interval(secs => expires_in) < NOW()').in_batches.delete_all
+    Doorkeeper::AccessToken.where.not(revoked_at: nil).where('revoked_at < NOW()').in_batches.delete_all
   end
 
   def vacuum_revoked_access_grants!
-    Doorkeeper::AccessGrant.where.not(revoked_at: nil).where('revoked_at < NOW()').delete_all
+    Doorkeeper::AccessGrant.where.not(expires_in: nil).where('created_at + make_interval(secs => expires_in) < NOW()').in_batches.delete_all
+    Doorkeeper::AccessGrant.where.not(revoked_at: nil).where('revoked_at < NOW()').in_batches.delete_all
   end
 end

app/lib/video_metadata_extractor.rb

@@ -43,6 +43,9 @@ class VideoMetadataExtractor
         @height      = video_stream[:height]
         @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
         @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
+        # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we
+        # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue.
+        @frame_rate ||= @r_frame_rate
       end
 
       if (audio_stream = audio_streams.first)

app/models/account.rb

@@ -61,9 +61,9 @@ class Account < ApplicationRecord
     trust_level
   )
 
-  USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
-  MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
-  URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
+  USERNAME_RE   = /[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?/i
+  MENTION_RE    = %r{(?<![=/[:word:]])@((#{USERNAME_RE})(?:@[[:word:].-]+[[:word:]]+)?)}i
+  URL_PREFIX_RE = %r{\Ahttp(s?)://[^/]+}
   USERNAME_ONLY_RE = /\A#{USERNAME_RE}\z/i
 
   include Attachmentable
@@ -112,10 +112,10 @@ class Account < ApplicationRecord
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) }
   scope :searchable, -> { without_unapproved.without_suspended.where(moved_to_account_id: nil) }
-  scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
+  scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).joins(:account_stat) }
   scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
-  scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
-  scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
+  scope :by_recent_status, -> { includes(:account_stat).merge(AccountStat.order('last_status_at DESC NULLS LAST')).references(:account_stat) }
+  scope :by_recent_sign_in, -> { order(Arel.sql('users.current_sign_in_at DESC NULLS LAST')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
   scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches("%.#{domain}"))) }
   scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) }

app/models/account_conversation.rb

@@ -16,34 +16,44 @@
 class AccountConversation < ApplicationRecord
   include Redisable
 
+  attr_writer :participant_accounts
+
+  before_validation :set_last_status
   after_commit :push_to_streaming_api
 
   belongs_to :account
   belongs_to :conversation
   belongs_to :last_status, class_name: 'Status'
 
-  before_validation :set_last_status
-
   def participant_account_ids=(arr)
     self[:participant_account_ids] = arr.sort
+    @participant_accounts = nil
   end
 
   def participant_accounts
-    if participant_account_ids.empty?
-      [account]
-    else
-      participants = Account.where(id: participant_account_ids)
-      participants.empty? ? [account] : participants
-    end
+    @participant_accounts ||= Account.where(id: participant_account_ids).to_a
+    @participant_accounts.presence || [account]
   end
 
   class << self
     def to_a_paginated_by_id(limit, options = {})
-      if options[:min_id]
-        paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
-      else
-        paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
+      array = begin
+        if options[:min_id]
+          paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
+        else
+          paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
+        end
       end
+
+      # Preload participants
+      participant_ids = array.flat_map(&:participant_account_ids)
+      accounts_by_id = Account.where(id: participant_ids).index_by(&:id)
+
+      array.each do |conversation|
+        conversation.participant_accounts = conversation.participant_account_ids.filter_map { |id| accounts_by_id[id] }
+      end
+
+      array
     end
 
     def paginate_by_min_id(limit, min_id = nil, max_id = nil)

app/models/admin/action_log_filter.rb

@@ -38,7 +38,7 @@ class Admin::ActionLogFilter
     destroy_status: { target_type: 'Status', action: 'destroy' }.freeze,
     destroy_user_role: { target_type: 'UserRole', action: 'destroy' }.freeze,
     destroy_canonical_email_block: { target_type: 'CanonicalEmailBlock', action: 'destroy' }.freeze,
-    disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze,
+    disable_2fa_user: { target_type: 'User', action: 'disable_2fa' }.freeze,
     disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze,
     disable_user: { target_type: 'User', action: 'disable' }.freeze,
     enable_custom_emoji: { target_type: 'CustomEmoji', action: 'enable' }.freeze,

app/models/admin/status_batch_action.rb

@@ -140,6 +140,6 @@ class Admin::StatusBatchAction
   end
 
   def allowed_status_ids
-    AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
+    Admin::AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
   end
 end

app/models/backup.rb

@@ -17,6 +17,6 @@
 class Backup < ApplicationRecord
   belongs_to :user, inverse_of: :backups
 
-  has_attached_file :dump, s3_permissions: 'private'
+  has_attached_file :dump, s3_permissions: ->(*) { ENV['S3_PERMISSION'] == '' ? nil : 'private' }
   do_not_validate_attachment_file_type :dump
 end

app/models/concerns/account_avatar.rb

@@ -18,7 +18,7 @@ module AccountAvatar
 
   included do
     # Avatar upload
-    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail]
+    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :avatar, less_than: LIMIT
     remotable_attachment :avatar, LIMIT, suppress_errors: false

app/models/concerns/account_header.rb

@@ -19,7 +19,7 @@ module AccountHeader
 
   included do
     # Header upload
-    has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail]
+    has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :header, less_than: LIMIT
     remotable_attachment :header, LIMIT, suppress_errors: false

app/models/concerns/attachmentable.rb

@@ -22,15 +22,14 @@ module Attachmentable
 
   included do
     def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName
-      options = { validate_media_type: false }.merge(options)
       super(name, options)
-      send(:"before_#{name}_post_process") do
+
+      send(:"before_#{name}_validate", prepend: true) do
         attachment = send(name)
         check_image_dimension(attachment)
         set_file_content_type(attachment)
         obfuscate_file_name(attachment)
         set_file_extension(attachment)
-        Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self)
       end
     end
   end
@@ -53,9 +52,13 @@ module Attachmentable
     return if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank?
 
     width, height = FastImage.size(attachment.queued_for_write[:original].path)
-    matrix_limit  = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT
+    return unless width.present? && height.present?
 
-    raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit)
+    if attachment.content_type == 'image/gif' && width * height > GIF_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} GIF files are not supported"
+    elsif width * height > MAX_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported"
+    end
   end
 
   def appropriate_extension(attachment)

app/models/concerns/ldap_authenticable.rb

@@ -6,7 +6,7 @@ module LdapAuthenticable
   class_methods do
     def authenticate_with_ldap(params = {})
       ldap   = Net::LDAP.new(ldap_options)
-      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: params[:email])
+      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: Net::LDAP::Filter.escape(params[:email]))
 
       if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password]))
         ldap_get_user(user_info.first)

app/models/concerns/omniauthable.rb

@@ -19,17 +19,18 @@ module Omniauthable
   end
 
   class_methods do
-    def find_for_oauth(auth, signed_in_resource = nil)
+    def find_for_omniauth(auth, signed_in_resource = nil)
       # EOLE-SSO Patch
       auth.uid = (auth.uid[0][:uid] || auth.uid[0][:user]) if auth.uid.is_a? Hashie::Array
-      identity = Identity.find_for_oauth(auth)
+      identity = Identity.find_for_omniauth(auth)
 
       # If a signed_in_resource is provided it always overrides the existing user
       # to prevent the identity being locked with accidentally created accounts.
       # Note that this may leave zombie accounts (with no associated identity) which
       # can be cleaned up at a later date.
       user   = signed_in_resource || identity.user
-      user ||= create_for_oauth(auth)
+      user ||= reattach_for_auth(auth)
+      user ||= create_for_auth(auth)
 
       if identity.user.nil?
         identity.user = user
@@ -39,19 +40,35 @@ module Omniauthable
       user
     end
 
-    def create_for_oauth(auth)
-      # Check if the user exists with provided email. If no email was provided,
+    private
+
+    def reattach_for_auth(auth)
+      # If allowed, check if a user exists with the provided email address,
+      # and return it if they does not have an associated identity with the
+      # current authentication provider.
+
+      # This can be used to provide a choice of alternative auth providers
+      # or provide smooth gradual transition between multiple auth providers,
+      # but this is discouraged because any insecure provider will put *all*
+      # local users at risk, regardless of which provider they registered with.
+
+      return unless ENV['ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH'] == 'true'
+
+      email, email_is_verified = email_from_auth(auth)
+      return unless email_is_verified
+
+      user = User.find_by(email: email)
+      return if user.nil? || Identity.exists?(provider: auth.provider, user_id: user.id)
+
+      user
+    end
+
+    def create_for_auth(auth)
+      # Create a user for the given auth params. If no email was provided,
       # we assign a temporary email and ask the user to verify it on
       # the next step via Auth::SetupController.show
 
-      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
-      assume_verified   = strategy&.security&.assume_email_is_verified
-      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
-      email             = auth.info.verified_email || auth.info.email
-
-      user = User.find_by(email: email) if email_is_verified
-
-      return user unless user.nil?
+      email, email_is_verified = email_from_auth(auth)
 
       user = User.new(user_params_from_auth(email, auth))
 
@@ -68,7 +85,14 @@ module Omniauthable
       user
     end
 
-    private
+    def email_from_auth(auth)
+      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
+      assume_verified   = strategy&.security&.assume_email_is_verified
+      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
+      email             = auth.info.verified_email || auth.info.email
+
+      [email, email_is_verified]
+    end
 
     def user_params_from_auth(email, auth)
       {

app/models/custom_emoji.rb

@@ -37,7 +37,7 @@ class CustomEmoji < ApplicationRecord
   belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
 
-  has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false
+  has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' } }, validate_media_type: false
 
   before_validation :downcase_domain
 

app/models/form/account_batch.rb

@@ -123,7 +123,18 @@ class Form::AccountBatch
       account: current_account,
       action: :suspend
     )
+
     Admin::SuspensionWorker.perform_async(account.id)
+
+    # Suspending a single account closes their associated reports, so
+    # mass-suspending would be consistent.
+    Report.where(target_account: account).unresolved.find_each do |report|
+      authorize(report, :update?)
+      log_action(:resolve, report)
+      report.resolve!(current_account)
+    rescue Mastodon::NotPermittedError
+      # This should not happen, but just in case, do not fail early
+    end
   end
 
   def approve_account(account)

app/models/identity.rb

@@ -12,11 +12,11 @@
 #
 
 class Identity < ApplicationRecord
-  belongs_to :user, dependent: :destroy
+  belongs_to :user
   validates :uid, presence: true, uniqueness: { scope: :provider }
   validates :provider, presence: true
 
-  def self.find_for_oauth(auth)
+  def self.find_for_omniauth(auth)
     find_or_create_by(uid: auth.uid, provider: auth.provider)
   end
 end

app/models/media_attachment.rb

@@ -169,7 +169,7 @@ class MediaAttachment < ApplicationRecord
   }.freeze
 
   GLOBAL_CONVERT_OPTIONS = {
-    all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date',
+    all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp',
   }.freeze
 
   belongs_to :account,          inverse_of: :media_attachments, optional: true

app/models/preview_card.rb

@@ -50,7 +50,7 @@ class PreviewCard < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy
 
-  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date' }, validate_media_type: false
+  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, validate_media_type: false
 
   validates :url, presence: true, uniqueness: true
   validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES

app/models/preview_card_provider.rb

@@ -25,7 +25,7 @@ class PreviewCardProvider < ApplicationRecord
 
   validates :domain, presence: true, uniqueness: true, domain: true
 
-  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false
+  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' } }, validate_media_type: false
   validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
   remotable_attachment :icon, LIMIT
 

app/models/relationship_filter.rb

@@ -60,13 +60,13 @@ class RelationshipFilter
   def relationship_scope(value)
     case value
     when 'following'
-      account.following.eager_load(:account_stat).reorder(nil)
+      account.following.includes(:account_stat).reorder(nil)
     when 'followed_by'
-      account.followers.eager_load(:account_stat).reorder(nil)
+      account.followers.includes(:account_stat).reorder(nil)
     when 'mutual'
-      account.followers.eager_load(:account_stat).reorder(nil).merge(Account.where(id: account.following))
+      account.followers.includes(:account_stat).reorder(nil).merge(Account.where(id: account.following))
     when 'invited'
-      Account.joins(user: :invite).merge(Invite.where(user: account.user)).eager_load(:account_stat).reorder(nil)
+      Account.joins(user: :invite).merge(Invite.where(user: account.user)).includes(:account_stat).reorder(nil)
     else
       raise Mastodon::InvalidParameterError, "Unknown relationship: #{value}"
     end
@@ -112,7 +112,7 @@ class RelationshipFilter
   def activity_scope(value)
     case value
     when 'dormant'
-      AccountStat.where(last_status_at: nil).or(AccountStat.where(AccountStat.arel_table[:last_status_at].lt(1.month.ago)))
+      Account.joins(:account_stat).where(account_stat: { last_status_at: [nil, ...1.month.ago] })
     else
       raise Mastodon::InvalidParameterError, "Unknown activity: #{value}"
     end

app/models/report.rb

@@ -39,7 +39,10 @@ class Report < ApplicationRecord
   scope :resolved,   -> { where.not(action_taken_at: nil) }
   scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) }
 
-  validates :comment, length: { maximum: 1_000 }
+  # A report is considered local if the reporter is local
+  delegate :local?, to: :account
+
+  validates :comment, length: { maximum: 1_000 }, if: :local?
   validates :rule_ids, absence: true, unless: :violation?
 
   validate :validate_rule_ids
@@ -50,10 +53,6 @@ class Report < ApplicationRecord
     violation: 2_000,
   }
 
-  def local?
-    false # Force uri_for to use uri attribute
-  end
-
   before_validation :set_uri, only: :create
 
   after_create_commit :trigger_webhooks

app/models/site_upload.rb

@@ -40,7 +40,7 @@ class SiteUpload < ApplicationRecord
     mascot: {}.freeze,
   }.freeze
 
-  has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
+  has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
 
   validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/
   validates :file, presence: true

app/models/status.rb

@@ -354,13 +354,25 @@ class Status < ApplicationRecord
 
       account_ids.uniq!
 
+      status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
+
       return if account_ids.empty?
 
       accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
 
+      status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
+
       cached_items.each do |item|
         item.account = accounts[item.account_id]
         item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
+
+        if item.reblog?
+          status_stat = status_stats[item.reblog.id]
+          item.reblog.status_stat = status_stat if status_stat.present?
+        else
+          status_stat = status_stats[item.id]
+          item.status_stat = status_stat if status_stat.present?
+        end
       end
     end
 

app/models/tag.rb

@@ -33,7 +33,7 @@ class Tag < ApplicationRecord
   HASTAG_LAST_SEQUENCE = '([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_NAME_PAT = "#{HASHTAG_FIRST_SEQUENCE}|#{HASTAG_LAST_SEQUENCE}"
 
-  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_PAT})/i
+  HASHTAG_RE = %r{(?<![=/)\w])#(#{HASHTAG_NAME_PAT})}i
   HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
   HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/
 

app/models/trends/statuses.rb

@@ -91,7 +91,7 @@ class Trends::Statuses < Trends::Base
   private
 
   def eligible?(status)
-    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
+    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && !status.account.sensitized? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
   end
 
   def calculate_scores(statuses, at_time)

app/models/user.rb

@@ -399,6 +399,16 @@ class User < ApplicationRecord
     Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch|
       batch.update_all(revoked_at: Time.now.utc)
       Web::PushSubscription.where(access_token_id: batch).delete_all
+
+      # Revoke each access token for the Streaming API, since `update_all``
+      # doesn't trigger ActiveRecord Callbacks:
+      # TODO: #28793 Combine into a single topic
+      payload = Oj.dump(event: :kill)
+      redis.pipelined do |pipeline|
+        batch.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
     end
   end
 

app/models/webhook.rb

@@ -20,6 +20,8 @@ class Webhook < ApplicationRecord
     report.created
   ).freeze
 
+  attr_writer :current_account
+
   scope :enabled, -> { where(enabled: true) }
 
   validates :url, presence: true, url: true
@@ -27,6 +29,7 @@ class Webhook < ApplicationRecord
   validates :events, presence: true
 
   validate :validate_events
+  validate :validate_permissions
 
   before_validation :strip_events
   before_validation :generate_secret
@@ -43,12 +46,29 @@ class Webhook < ApplicationRecord
     update!(enabled: false)
   end
 
+  def required_permissions
+    events.map { |event| Webhook.permission_for_event(event) }
+  end
+
+  def self.permission_for_event(event)
+    case event
+    when 'account.approved', 'account.created', 'account.updated'
+      :manage_users
+    when 'report.created'
+      :manage_reports
+    end
+  end
+
   private
 
   def validate_events
     errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) }
   end
 
+  def validate_permissions
+    errors.add(:events, :invalid_permissions) if defined?(@current_account) && required_permissions.any? { |permission| !@current_account.user_role.can?(permission) }
+  end
+
   def strip_events
     self.events = events.map { |str| str.strip.presence }.compact if events.present?
   end

app/policies/admin/status_policy.rb

@@ -12,7 +12,7 @@ class Admin::StatusPolicy < ApplicationPolicy
   end
 
   def show?
-    role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported?)
+    role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported? || viewable_through_normal_policy?)
   end
 
   def destroy?
@@ -26,4 +26,10 @@ class Admin::StatusPolicy < ApplicationPolicy
   def review?
     role.can?(:manage_taxonomies)
   end
+
+  private
+
+  def viewable_through_normal_policy?
+    StatusPolicy.new(current_account, record, @preloaded_relations).show?
+  end
 end

app/policies/backup_policy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class BackupPolicy < ApplicationPolicy
-  MIN_AGE = 1.week
+  MIN_AGE = 6.days
 
   def create?
     user_signed_in? && current_user.backups.where('created_at >= ?', MIN_AGE.ago).count.zero?

app/policies/webhook_policy.rb

@@ -14,7 +14,7 @@ class WebhookPolicy < ApplicationPolicy
   end
 
   def update?
-    role.can?(:manage_webhooks)
+    role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
   end
 
   def enable?
@@ -30,6 +30,6 @@ class WebhookPolicy < ApplicationPolicy
   end
 
   def destroy?
-    role.can?(:manage_webhooks)
+    role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
   end
 end

app/serializers/rest/featured_tag_serializer.rb

@@ -10,7 +10,9 @@ class REST::FeaturedTagSerializer < ActiveModel::Serializer
   end
 
   def url
-    short_account_tag_url(object.account, object.tag)
+    # The path is hardcoded because we have to deal with both local and
+    # remote users, which are different routes
+    account_with_domain_url(object.account, "tagged/#{object.tag.to_param}")
   end
 
   def name

app/serializers/rest/preview_card_serializer.rb

@@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer
   def image
     object.image? ? full_asset_url(object.image.url(:original)) : nil
   end
+
+  def html
+    Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED)
+  end
 end

app/services/activitypub/fetch_featured_collection_service.rb

@@ -23,9 +23,9 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 

app/services/activitypub/fetch_remote_account_service.rb

@@ -2,7 +2,7 @@
 
 class ActivityPub::FetchRemoteAccountService < ActivityPub::FetchRemoteActorService
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
+  def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
     actor = super
     return actor if actor.nil? || actor.is_a?(Account)
 

app/services/activitypub/fetch_remote_actor_service.rb

@@ -10,15 +10,15 @@ class ActivityPub::FetchRemoteActorService < BaseService
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
 
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
+  def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
     return if domain_not_allowed?(uri)
     return ActivityPub::TagManager.instance.uri_to_actor(uri) if ActivityPub::TagManager.instance.local_uri?(uri)
 
     @json = begin
       if prefetched_body.nil?
-        fetch_resource(uri, id)
+        fetch_resource(uri, true)
       else
-        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+        body_to_json(prefetched_body, compare_id: uri)
       end
     rescue Oj::ParseError
       raise Error, "Error parsing JSON-LD document #{uri}"

app/services/activitypub/fetch_remote_key_service.rb

@@ -6,23 +6,10 @@ class ActivityPub::FetchRemoteKeyService < BaseService
   class Error < StandardError; end
 
   # Returns actor that owns the key
-  def call(uri, id: true, prefetched_body: nil, suppress_errors: true)
+  def call(uri, suppress_errors: true)
     raise Error, 'No key URI given' if uri.blank?
 
-    if prefetched_body.nil?
-      if id
-        @json = fetch_resource_without_id_validation(uri)
-        if actor_type?
-          @json = fetch_resource(@json['id'], true)
-        elsif uri != @json['id']
-          raise Error, "Fetched URI #{uri} has wrong id #{@json['id']}"
-        end
-      else
-        @json = fetch_resource(uri, id)
-      end
-    else
-      @json = body_to_json(prefetched_body, compare_id: id ? uri : nil)
-    end
+    @json = fetch_resource(uri, false)
 
     raise Error, "Unable to fetch key JSON at #{uri}" if @json.nil?
     raise Error, "Unsupported JSON-LD context for document #{uri}" unless supported_context?(@json)

app/services/activitypub/fetch_remote_poll_service.rb

@@ -8,6 +8,6 @@ class ActivityPub::FetchRemotePollService < BaseService
 
     return unless supported_context?(json)
 
-    ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json)
+    ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json, json)
   end
 end

app/services/activitypub/fetch_remote_status_service.rb

@@ -7,13 +7,13 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   DISCOVERIES_PER_REQUEST = 1000
 
   # Should be called when uri has already been checked for locality
-  def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
+  def call(uri, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
     @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}"
     @json = begin
       if prefetched_body.nil?
-        fetch_resource(uri, id, on_behalf_of)
+        fetch_resource(uri, true, on_behalf_of)
       else
-        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+        body_to_json(prefetched_body, compare_id: uri)
       end
     end
 
@@ -63,7 +63,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
   def account_from_uri(uri)
     actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
-    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true, request_id: @request_id) if actor.nil? || actor.possibly_stale?
+    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, request_id: @request_id) if actor.nil? || actor.possibly_stale?
     actor
   end
 

app/services/activitypub/fetch_replies_service.rb

@@ -26,9 +26,9 @@ class ActivityPub::FetchRepliesService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 
@@ -36,7 +36,21 @@ class ActivityPub::FetchRepliesService < BaseService
     return collection_or_uri if collection_or_uri.is_a?(Hash)
     return unless @allow_synchronous_requests
     return if invalid_origin?(collection_or_uri)
-    fetch_resource_without_id_validation(collection_or_uri, nil, true)
+
+    # NOTE: For backward compatibility reasons, Mastodon signs outgoing
+    # queries incorrectly by default.
+    #
+    # While this is relevant for all URLs with query strings, this is
+    # the only code path where this happens in practice.
+    #
+    # Therefore, retry with correct signatures if this fails.
+    begin
+      fetch_resource_without_id_validation(collection_or_uri, nil, true)
+    rescue Mastodon::UnexpectedResponseError => e
+      raise unless e.response && e.response.code == 401 && Addressable::URI.parse(collection_or_uri).query.present?
+
+      fetch_resource_without_id_validation(collection_or_uri, nil, true, request_options: { with_query_string: true })
+    end
   end
 
   def filtered_replies

app/services/activitypub/process_account_service.rb

@@ -76,6 +76,9 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.suspended_at      = domain_block.created_at if auto_suspend?
     @account.suspension_origin = :local if auto_suspend?
     @account.silenced_at       = domain_block.created_at if auto_silence?
+
+    set_immediate_protocol_attributes!
+
     @account.save
   end
 
@@ -271,7 +274,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def moved_account
     account   = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account)
-    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true, request_id: @options[:request_id])
+    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], break_on_redirect: true, request_id: @options[:request_id])
     account
   end
 

app/services/activitypub/process_status_update_service.rb

@@ -5,10 +5,11 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   include Redisable
   include Lockable
 
-  def call(status, json, request_id: nil)
+  def call(status, activity_json, object_json, request_id: nil)
     raise ArgumentError, 'Status has unsaved changes' if status.changed?
 
-    @json                      = json
+    @activity_json             = activity_json
+    @json                      = object_json
     @status_parser             = ActivityPub::Parser::StatusParser.new(@json)
     @uri                       = @status_parser.uri
     @status                    = status
@@ -308,6 +309,6 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   end
 
   def forwarder
-    @forwarder ||= ActivityPub::Forwarder.new(@account, @json, @status)
+    @forwarder ||= ActivityPub::Forwarder.new(@account, @activity_json, @status)
   end
 end

app/services/activitypub/synchronize_followers_service.rb

@@ -59,9 +59,9 @@ class ActivityPub::SynchronizeFollowersService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 

app/services/fetch_oembed_service.rb

@@ -100,7 +100,7 @@ class FetchOEmbedService
   end
 
   def validate(oembed)
-    oembed if oembed[:version].to_s == '1.0' && oembed[:type].present?
+    oembed if oembed.present? && oembed[:version].to_s == '1.0' && oembed[:type].present?
   end
 
   def html

app/services/fetch_resource_service.rb

@@ -43,11 +43,19 @@ class FetchResourceService < BaseService
     @response_code = response.code
     return nil if response.code != 200
 
-    if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
+    if valid_activitypub_content_type?(response)
       body = response.body_with_limit
       json = body_to_json(body)
 
-      [json['id'], { prefetched_body: body, id: true }] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json))
+      return unless supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json))
+
+      if json['id'] != @url
+        return if terminal
+
+        return process(json['id'], terminal: true)
+      end
+
+      [@url, { prefetched_body: body }]
     elsif !terminal
       link_header = response['Link'] && parse_link_header(response)
 

app/services/follow_service.rb

@@ -71,7 +71,7 @@ class FollowService < BaseService
     if @target_account.local?
       LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
     elsif @target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url)
+      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, { 'bypass_availability' => true })
     end
 
     follow_request

app/services/keys/query_service.rb

@@ -69,7 +69,7 @@ class Keys::QueryService < BaseService
 
     return if json['items'].blank?
 
-    @devices = json['items'].map do |device|
+    @devices = as_array(json['items']).map do |device|
       Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim'])
     end
   rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e