Bump version to 3.4.7

Browsers are phasing out X-XSS-Protection, but Safari and IE still support it.

.circleci/config.yml

@@ -167,8 +167,45 @@ jobs:
           name: Create database
           command: ./bin/rails db:create
       - run:
-          name: Run migrations
+          command: ./bin/rails db:migrate VERSION=20171010025614
+          name: Run migrations up to v2.0.0
+      - run:
+          command: ./bin/rails tests:migrations:populate_v2
+          name: Populate database with test data
+      - run:
           command: ./bin/rails db:migrate
+          name: Run all remaining migrations
+
+  test-two-step-migrations:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:2.7-buster-node
+        environment: *ruby_environment
+      - image: circleci/postgres:12.2
+        environment:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+      - image: circleci/redis:5-alpine
+    steps:
+      - *attach_workspace
+      - *install_system_dependencies
+      - run:
+          command: ./bin/rails db:create
+          name: Create database
+      - run:
+          command: ./bin/rails db:migrate VERSION=20171010025614
+          name: Run migrations up to v2.0.0
+      - run:
+          command: ./bin/rails tests:migrations:populate_v2
+          name: Populate database with test data
+      - run:
+          command: ./bin/rails db:migrate
+          name: Run all pre-deployment migrations
+          evironment:
+            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
+      - run:
+          command: ./bin/rails db:migrate
+          name: Run all post-deployment remaining migrations
 
   test-ruby2.7:
     <<: *defaults
@@ -238,6 +275,9 @@ workflows:
       - test-migrations:
           requires:
             - install-ruby2.7
+      - test-two-step-migrations:
+          requires:
+            - install-ruby2.7
       - test-ruby2.7:
           requires:
             - install-ruby2.7

.github/workflows/build-image.yml

@@ -0,0 +1,34 @@
+name: Build container image
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "*"
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/metadata-action@v3
+        id: meta
+        with:
+          images: tootsuite/mastodon
+          flavor: |
+            latest=auto
+          tags: |
+            type=edge,branch=main
+            type=semver,pattern={{ raw }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=tootsuite/mastodon:latest
+          cache-to: type=inline

CHANGELOG.md

@@ -3,6 +3,53 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [3.4.7] - 2022-03-30
+### Security
+
+- Fix being able to post URLs longer than 4096 characters ([Gargron](https://github.com/mastodon/mastodon/pull/17908))
+- Fix being able to bypass e-mail restrictions ([Gargron](https://github.com/mastodon/mastodon/pull/17909))
+
+## [3.4.6] - 2022-02-03
+### Fixed
+- Fix `mastodon:webpush:generate_vapid_key` task requiring a functional environment ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17338))
+- Fix spurious errors when receiving an Add activity for a private post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17425))
+
+### Security
+- Fix error-prone SQL queries ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/15828))
+- Fix not compacting incoming signed JSON-LD activities ([puckipedia](https://github.com/mastodon/mastodon/pull/17426), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17428)) (CVE-2022-24307)
+- Fix insufficient sanitization of report comments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17430))
+- Fix stop condition of a Common Table Expression ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17427))
+- Disable legacy XSS filtering ([Wonderfall](https://github.com/mastodon/mastodon/pull/17289))
+
+## [3.4.5] - 2022-01-31
+### Added
+- Add more advanced migration tests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17393))
+- Add github workflow to build Docker images ([unasuke](https://github.com/mastodon/mastodon/pull/16973), [Gargron](https://github.com/mastodon/mastodon/pull/16980), [Gargron](https://github.com/mastodon/mastodon/pull/17000))
+
+### Fixed
+- Fix some old migrations failing when skipping releases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17394))
+- Fix migrations script failing in certain edge cases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17398))
+- Fix Docker build ([tribela](https://github.com/mastodon/mastodon/pull/17188))
+- Fix Ruby 3.0 dependencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16723))
+- Fix followers synchronization mechanism ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16510))
+
+## [3.4.4] - 2021-11-26
+### Fixed
+
+- Fix error when suspending user with an already blocked canonical email ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17036))
+- Fix overflow of long profile fields in admin UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17010))
+- Fix confusing error when WebFinger request returns empty document ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16986))
+- Fix upload of remote media with OpenStack Swift sometimes failing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16998))
+- Fix logout link not working in Safari ([noellabo](https://github.com/mastodon/mastodon/pull/16574))
+- Fix “open” link of media modal not closing modal in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16524))
+- Fix replying from modal in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16516))
+- Fix `mastodon:setup` command crashing in some circumstances ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16976))
+
+### Security
+
+- Fix filtering DMs from non-followed users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17042))
+- Fix handling of recursive toots in WebUI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17041))
+
 ## [3.4.3] - 2021-11-06
 ### Fixed
 

Dockerfile

@@ -54,8 +54,8 @@ RUN npm install -g yarn && \
 COPY Gemfile* package.json yarn.lock /opt/mastodon/
 
 RUN cd /opt/mastodon && \
-  bundle config set deployment 'true' && \
-  bundle config set without 'development test' && \
+  bundle config set --local deployment 'true' && \
+  bundle config set --local without 'development test' && \
 	bundle install -j"$(nproc)" && \
 	yarn install --pure-lockfile
 

Gemfile.lock

@@ -545,8 +545,9 @@ GEM
       rack (>= 1.1)
       rubocop (>= 1.7.0, < 2.0)
     ruby-progressbar (1.11.0)
-    ruby-saml (1.11.0)
-      nokogiri (>= 1.5.10)
+    ruby-saml (1.13.0)
+      nokogiri (>= 1.10.5)
+      rexml
     ruby2_keywords (0.0.4)
     rufus-scheduler (3.6.0)
       fugit (~> 1.1, >= 1.1.6)

app/helpers/context_helper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module ContextHelper
+  NAMED_CONTEXT_MAP = {
+    activitystreams: 'https://www.w3.org/ns/activitystreams',
+    security: 'https://w3id.org/security/v1',
+  }.freeze
+
+  CONTEXT_EXTENSION_MAP = {
+    manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
+    sensitive: { 'sensitive' => 'as:sensitive' },
+    hashtag: { 'Hashtag' => 'as:Hashtag' },
+    moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
+    also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
+    emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
+    featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
+    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
+    atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
+    conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
+    focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
+    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
+    blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
+    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
+    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
+    olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
+    suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
+  }.freeze
+
+  def full_context
+    serialized_context(NAMED_CONTEXT_MAP, CONTEXT_EXTENSION_MAP)
+  end
+
+  def serialized_context(named_contexts_map, context_extensions_map)
+    context_array = []
+
+    named_contexts     = named_contexts_map.keys
+    context_extensions = context_extensions_map.keys
+
+    named_contexts.each do |key|
+      context_array << NAMED_CONTEXT_MAP[key]
+    end
+
+    extensions = context_extensions.each_with_object({}) do |key, h|
+      h.merge!(CONTEXT_EXTENSION_MAP[key])
+    end
+
+    context_array << extensions unless extensions.empty?
+
+    if context_array.size == 1
+      context_array.first
+    else
+      context_array
+    end
+  end
+end

app/helpers/jsonld_helper.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module JsonLdHelper
+  include ContextHelper
+
   def equals_or_includes?(haystack, needle)
     haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle
   end
@@ -63,6 +65,84 @@ module JsonLdHelper
     graph.dump(:normalize)
   end
 
+  def compact(json)
+    compacted = JSON::LD::API.compact(json.without('signature'), full_context, documentLoader: method(:load_jsonld_context))
+    compacted['signature'] = json['signature']
+    compacted
+  end
+
+  # Patches a JSON-LD document to avoid compatibility issues on redistribution
+  #
+  # Since compacting a JSON-LD document against Mastodon's built-in vocabulary
+  # means other extension namespaces will be expanded, malformed JSON-LD
+  # attributes lost, and some values “unexpectedly” compacted this method
+  # patches the following likely sources of incompatibility:
+  # - 'https://www.w3.org/ns/activitystreams#Public' being compacted to
+  #   'as:Public' (for instance, pre-3.4.0 Mastodon does not understand
+  #   'as:Public')
+  # - single-item arrays being compacted to the item itself (`[foo]` being
+  #   compacted to `foo`)
+  #
+  # It is not always possible for `patch_for_forwarding!` to produce a document
+  # deemed safe for forwarding. Use `safe_for_forwarding?` to check the status
+  # of the output document.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [void]
+  def patch_for_forwarding!(original, compacted)
+    original.without('@context', 'signature').each do |key, value|
+      next if value.nil? || !compacted.key?(key)
+
+      compacted_value = compacted[key]
+      if value.is_a?(Hash) && compacted_value.is_a?(Hash)
+        patch_for_forwarding!(value, compacted_value)
+      elsif value.is_a?(Array)
+        compacted_value = [compacted_value] unless compacted_value.is_a?(Array)
+        return if value.size != compacted_value.size
+
+        compacted[key] = value.zip(compacted_value).map do |v, vc|
+          if v.is_a?(Hash) && vc.is_a?(Hash)
+            patch_for_forwarding!(v, vc)
+            vc
+          elsif v == 'https://www.w3.org/ns/activitystreams#Public' && vc == 'as:Public'
+            v
+          else
+            vc
+          end
+        end
+      elsif value == 'https://www.w3.org/ns/activitystreams#Public' && compacted_value == 'as:Public'
+        compacted[key] = value
+      end
+    end
+  end
+
+  # Tests whether a JSON-LD compaction is deemed safe for redistribution,
+  # that is, if it doesn't change its meaning to consumers that do not actually
+  # handle JSON-LD, but rely on values being serialized in a certain way.
+  #
+  # See `patch_for_forwarding!` for details.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [Boolean] Whether the patched document is deemed safe
+  def safe_for_forwarding?(original, compacted)
+    original.without('@context', 'signature').all? do |key, value|
+      compacted_value = compacted[key]
+      return false unless value.class == compacted_value.class
+
+      if value.is_a?(Hash)
+        safe_for_forwarding?(value, compacted_value)
+      elsif value.is_a?(Array)
+        value.zip(compacted_value).all? do |v, vc|
+          v.is_a?(Hash) ? (vc.is_a?(Hash) && safe_for_forwarding?(v, vc)) : v == vc
+        end
+      else
+        value == compacted_value
+      end
+    end
+  end
+
   def fetch_resource(uri, id, on_behalf_of = nil)
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)

app/javascript/mastodon/components/modal_root.js

@@ -6,6 +6,10 @@ import { multiply } from 'color-blend';
 
 export default class ModalRoot extends React.PureComponent {
 
+  static contextTypes = {
+    router: PropTypes.object,
+  };
+
   static propTypes = {
     children: PropTypes.node,
     onClose: PropTypes.func.isRequired,

app/javascript/mastodon/features/compose/containers/navigation_container.js

@@ -21,6 +21,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     dispatch(openModal('CONFIRM', {
       message: intl.formatMessage(messages.logoutMessage),
       confirm: intl.formatMessage(messages.logoutConfirm),
+      closeWhenConfirm: false,
       onConfirm: () => logOut(),
     }));
   },

app/javascript/mastodon/features/compose/index.js

@@ -74,6 +74,7 @@ class Compose extends React.PureComponent {
     dispatch(openModal('CONFIRM', {
       message: intl.formatMessage(messages.logoutMessage),
       confirm: intl.formatMessage(messages.logoutConfirm),
+      closeWhenConfirm: false,
       onConfirm: () => logOut(),
     }));
 

app/javascript/mastodon/features/picture_in_picture/components/footer.js

@@ -114,7 +114,11 @@ class Footer extends ImmutablePureComponent {
       return;
     }
 
-    const { status } = this.props;
+    const { status, onClose } = this.props;
+
+    if (onClose) {
+      onClose();
+    }
 
     router.history.push(`/statuses/${status.get('id')}`);
   }

app/javascript/mastodon/features/status/index.js

@@ -83,7 +83,7 @@ const makeMapStateToProps = () => {
     ancestorsIds = ancestorsIds.withMutations(mutable => {
       let id = statusId;
 
-      while (id) {
+      while (id && !mutable.includes(id)) {
         mutable.unshift(id);
         id = inReplyTos.get(id);
       }
@@ -101,7 +101,7 @@ const makeMapStateToProps = () => {
     const ids = [statusId];
 
     while (ids.length > 0) {
-      let id        = ids.shift();
+      let id        = ids.pop();
       const replies = contextReplies.get(id);
 
       if (statusId !== id) {
@@ -110,7 +110,7 @@ const makeMapStateToProps = () => {
 
       if (replies) {
         replies.reverse().forEach(reply => {
-          ids.unshift(reply);
+          if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply);
         });
       }
     }

app/javascript/mastodon/features/ui/components/confirmation_modal.js

@@ -13,15 +13,22 @@ class ConfirmationModal extends React.PureComponent {
     onConfirm: PropTypes.func.isRequired,
     secondary: PropTypes.string,
     onSecondary: PropTypes.func,
+    closeWhenConfirm: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
 
+  static defaultProps = {
+    closeWhenConfirm: true,
+  };
+
   componentDidMount() {
     this.button.focus();
   }
 
   handleClick = () => {
-    this.props.onClose();
+    if (this.props.closeWhenConfirm) {
+      this.props.onClose();
+    }
     this.props.onConfirm();
   }
 

app/javascript/mastodon/features/ui/components/link_footer.js

@@ -17,6 +17,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     dispatch(openModal('CONFIRM', {
       message: intl.formatMessage(messages.logoutMessage),
       confirm: intl.formatMessage(messages.logoutConfirm),
+      closeWhenConfirm: false,
       onConfirm: () => logOut(),
     }));
   },

app/javascript/styles/mastodon/admin.scss

@@ -829,6 +829,7 @@ a.name-tag,
     padding: 0 5px;
     margin-bottom: 10px;
     flex: 1 0 50%;
+    max-width: 100%;
   }
 
   .account__header__fields,

app/lib/activitypub/activity/add.rb

@@ -7,7 +7,7 @@ class ActivityPub::Activity::Add < ActivityPub::Activity
     status   = status_from_uri(object_uri)
     status ||= fetch_remote_original_status
 
-    return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
+    return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status) && status.distributable?
 
     StatusPin.create!(account: @account, status: status)
   end

app/lib/activitypub/adapter.rb

@@ -1,30 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
-  NAMED_CONTEXT_MAP = {
-    activitystreams: 'https://www.w3.org/ns/activitystreams',
-    security: 'https://w3id.org/security/v1',
-  }.freeze
-
-  CONTEXT_EXTENSION_MAP = {
-    manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
-    sensitive: { 'sensitive' => 'as:sensitive' },
-    hashtag: { 'Hashtag' => 'as:Hashtag' },
-    moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
-    also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
-    emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
-    featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
-    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
-    atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
-    conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
-    focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
-    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
-    blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
-    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
-    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
-    olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
-    suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
-  }.freeze
+  include ContextHelper
 
   def self.default_key_transform
     :camel_lower
@@ -35,7 +12,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   end
 
   def serializable_hash(options = nil)
-    named_contexts     = {}
+    named_contexts     = { activitystreams: NAMED_CONTEXT_MAP['activitystreams'] }
     context_extensions = {}
 
     options         = serialization_options(options)
@@ -45,29 +22,4 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
 
     { '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash)
   end
-
-  private
-
-  def serialized_context(named_contexts_map, context_extensions_map)
-    context_array = []
-
-    named_contexts     = [:activitystreams] + named_contexts_map.keys
-    context_extensions = context_extensions_map.keys
-
-    named_contexts.each do |key|
-      context_array << NAMED_CONTEXT_MAP[key]
-    end
-
-    extensions = context_extensions.each_with_object({}) do |key, h|
-      h.merge!(CONTEXT_EXTENSION_MAP[key])
-    end
-
-    context_array << extensions unless extensions.empty?
-
-    if context_array.size == 1
-      context_array.first
-    else
-      context_array
-    end
-  end
 end

app/lib/extractor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Extractor
+  MAX_DOMAIN_LENGTH = 253
+
   extend Twitter::TwitterText::Extractor
 
   module_function
@@ -14,7 +16,12 @@ module Extractor
     text.to_s.scan(Account::MENTION_RE) do |screen_name, _|
       match_data = $LAST_MATCH_INFO
       after = $'
+
       unless Twitter::TwitterText::Regex[:end_mention_match].match?(after)
+        username, domain = screen_name.split('@')
+
+        next if domain.present? && domain.length > MAX_DOMAIN_LENGTH
+
         start_position = match_data.char_begin(1) - 1
         end_position = match_data.char_end(1)
         possible_entries << {

app/lib/webfinger.rb

@@ -46,7 +46,9 @@ class Webfinger
   def body_from_webfinger(url = standard_url, use_fallback = true)
     webfinger_request(url).perform do |res|
       if res.code == 200
-        res.body_with_limit
+        body = res.body_with_limit
+        raise Webfinger::Error, "Request for #{@uri} returned empty response" if body.empty?
+        body
       elsif res.code == 404 && use_fallback
         body_from_host_meta
       elsif res.code == 410

app/models/account.rb

@@ -60,6 +60,7 @@ class Account < ApplicationRecord
 
   USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
+  URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
 
   include AccountAssociations
   include AccountAvatar
@@ -379,7 +380,7 @@ class Account < ApplicationRecord
   def synchronization_uri_prefix
     return 'local' if local?
 
-    @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//]
+    @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
   end
 
   class Field < ActiveModelSerializers::Model
@@ -425,6 +426,9 @@ class Account < ApplicationRecord
   end
 
   class << self
+    DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze
+    TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
+
     def readonly_attributes
       super - %w(statuses_count following_count followers_count)
     end
@@ -435,70 +439,29 @@ class Account < ApplicationRecord
     end
 
     def search_for(terms, limit = 10, offset = 0)
-      textsearch, query = generate_query_for_search(terms)
+      tsquery = generate_query_for_search(terms)
 
       sql = <<-SQL.squish
         SELECT
           accounts.*,
-          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
+          ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
         FROM accounts
-        WHERE #{query} @@ #{textsearch}
+        WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
           AND accounts.suspended_at IS NULL
           AND accounts.moved_to_account_id IS NULL
         ORDER BY rank DESC
-        LIMIT ? OFFSET ?
+        LIMIT :limit OFFSET :offset
       SQL
 
-      records = find_by_sql([sql, limit, offset])
+      records = find_by_sql([sql, limit: limit, offset: offset, tsquery: tsquery])
       ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
       records
     end
 
     def advanced_search_for(terms, account, limit = 10, following = false, offset = 0)
-      textsearch, query = generate_query_for_search(terms)
-
-      if following
-        sql = <<-SQL.squish
-          WITH first_degree AS (
-            SELECT target_account_id
-            FROM follows
-            WHERE account_id = ?
-            UNION ALL
-            SELECT ?
-          )
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)
-          WHERE accounts.id IN (SELECT * FROM first_degree)
-            AND #{query} @@ #{textsearch}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id
-          ORDER BY rank DESC
-          LIMIT ? OFFSET ?
-        SQL
-
-        records = find_by_sql([sql, account.id, account.id, account.id, limit, offset])
-      else
-        sql = <<-SQL.squish
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)
-          WHERE #{query} @@ #{textsearch}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id
-          ORDER BY rank DESC
-          LIMIT ? OFFSET ?
-        SQL
-
-        records = find_by_sql([sql, account.id, account.id, limit, offset])
-      end
-
+      tsquery = generate_query_for_search(terms)
+      sql = advanced_search_for_sql_template(following)
+      records = find_by_sql([sql, id: account.id, limit: limit, offset: offset, tsquery: tsquery])
       ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
       records
     end
@@ -520,12 +483,55 @@ class Account < ApplicationRecord
 
     private
 
-    def generate_query_for_search(terms)
-      terms      = Arel.sql(connection.quote(terms.gsub(/['?\\:]/, ' ')))
-      textsearch = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
-      query      = "to_tsquery('simple', ''' ' || #{terms} || ' ''' || ':*')"
+    def generate_query_for_search(unsanitized_terms)
+      terms = unsanitized_terms.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ')
 
-      [textsearch, query]
+      # The final ":*" is for prefix search.
+      # The trailing space does not seem to fit any purpose, but `to_tsquery`
+      # behaves differently with and without a leading space if the terms start
+      # with `./`, `../`, or `.. `. I don't understand why, so, in doubt, keep
+      # the same query.
+      "' #{terms} ':*"
+    end
+
+    def advanced_search_for_sql_template(following)
+      if following
+        <<-SQL.squish
+          WITH first_degree AS (
+            SELECT target_account_id
+            FROM follows
+            WHERE account_id = :id
+            UNION ALL
+            SELECT :id
+          )
+          SELECT
+            accounts.*,
+            (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
+          FROM accounts
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id)
+          WHERE accounts.id IN (SELECT * FROM first_degree)
+            AND to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
+            AND accounts.suspended_at IS NULL
+            AND accounts.moved_to_account_id IS NULL
+          GROUP BY accounts.id
+          ORDER BY rank DESC
+          LIMIT :limit OFFSET :offset
+        SQL
+      else
+        <<-SQL.squish
+          SELECT
+            accounts.*,
+            (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
+          FROM accounts
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id)
+          WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
+            AND accounts.suspended_at IS NULL
+            AND accounts.moved_to_account_id IS NULL
+          GROUP BY accounts.id
+          ORDER BY rank DESC
+          LIMIT :limit OFFSET :offset
+        SQL
+      end
     end
   end
 

app/models/canonical_email_block.rb

@@ -15,7 +15,7 @@ class CanonicalEmailBlock < ApplicationRecord
 
   belongs_to :reference_account, class_name: 'Account'
 
-  validates :canonical_email_hash, presence: true
+  validates :canonical_email_hash, presence: true, uniqueness: true
 
   def email=(email)
     self.canonical_email_hash = email_to_canonical_email_hash(email)

app/models/concerns/account_interactions.rb

@@ -251,10 +251,13 @@ module AccountInteractions
          .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
   end
 
-  def remote_followers_hash(url_prefix)
-    Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do
+  def remote_followers_hash(url)
+    url_prefix = url[Account::URL_PREFIX_RE]
+    return if url_prefix.blank?
+
+    Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do
       digest = "\x00" * 32
-      followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri|
+      followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri|
         Xorcist.xor!(digest, Digest::SHA256.digest(uri))
       end
       digest.unpack('H*')[0]

app/models/status.rb

@@ -96,15 +96,12 @@ class Status < ApplicationRecord
   scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) }
   scope :not_domain_blocked_by_account, ->(account) { account.excluded_from_timeline_domains.blank? ? left_outer_joins(:account) : left_outer_joins(:account).where('accounts.domain IS NULL OR accounts.domain NOT IN (?)', account.excluded_from_timeline_domains) }
   scope :tagged_with_all, ->(tag_ids) {
-    Array(tag_ids).reduce(self) do |result, id|
+    Array(tag_ids).map(&:to_i).reduce(self) do |result, id|
       result.joins("INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}")
     end
   }
   scope :tagged_with_none, ->(tag_ids) {
-    Array(tag_ids).reduce(self) do |result, id|
-      result.joins("LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}")
-            .where("t#{id}.tag_id IS NULL")
-    end
+    where('NOT EXISTS (SELECT * FROM statuses_tags forbidden WHERE forbidden.status_id = statuses.id AND forbidden.tag_id IN (?))', tag_ids)
   }
 
   cache_associated :application,

app/models/user.rb

@@ -86,11 +86,11 @@ class User < ApplicationRecord
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
-  validates_with BlacklistedEmailValidator, on: :create
+  validates_with BlacklistedEmailValidator, if: -> { !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
 
-  # Those are honeypot/antispam fields
+  # Honeypot/anti-spam fields
   attr_accessor :registration_form_time, :website, :confirm_password
 
   validates_with RegistrationFormTimeValidator, on: :create

app/services/activitypub/process_collection_service.rb

@@ -5,11 +5,27 @@ class ActivityPub::ProcessCollectionService < BaseService
 
   def call(body, account, **options)
     @account = account
-    @json    = Oj.load(body, mode: :strict)
+    @json    = original_json = Oj.load(body, mode: :strict)
     @options = options
 
+    begin
+      @json = compact(@json) if @json['signature'].is_a?(Hash)
+    rescue JSON::LD::JsonLdError => e
+      Rails.logger.debug "Error when compacting JSON-LD document for #{value_or_id(@json['actor'])}: #{e.message}"
+      @json = original_json.without('signature')
+    end
+
     return if !supported_context? || (different_actor? && verify_account!.nil?) || suspended_actor? || @account.local?
 
+    if @json['signature'].present?
+      # We have verified the signature, but in the compaction step above, might
+      # have introduced incompatibilities with other servers that do not
+      # normalize the JSON-LD documents (for instance, previous Mastodon
+      # versions), so skip redistribution if we can't get a safe document.
+      patch_for_forwarding!(original_json, @json)
+      @json.delete('signature') unless safe_for_forwarding?(original_json, @json)
+    end
+
     case @json['type']
     when 'Collection', 'CollectionPage'
       process_items @json['items']

app/services/notify_service.rb

@@ -67,8 +67,53 @@ class NotifyService < BaseService
     message? && @notification.target_status.direct_visibility?
   end
 
+  # Returns true if the sender has been mentionned by the recipient up the thread
   def response_to_recipient?
-    @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility?
+    return false if @notification.target_status.in_reply_to_id.nil?
+
+    # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads
+    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero?
+      WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender, path) AS (
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END),
+            ARRAY[s.id]
+          FROM statuses s
+          WHERE s.id = :id
+        UNION ALL
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END),
+            st.path || s.id
+          FROM ancestors st
+          JOIN statuses s ON s.id = st.in_reply_to_id
+          WHERE st.replying_to_sender IS FALSE AND NOT s.id = ANY(path)
+      )
+      SELECT COUNT(*)
+      FROM ancestors st
+      JOIN statuses s ON s.id = st.id
+      WHERE st.replying_to_sender IS TRUE AND s.visibility = 3
+    SQL
   end
 
   def from_staff?

app/validators/status_length_validator.rb

@@ -3,35 +3,57 @@
 class StatusLengthValidator < ActiveModel::Validator
   MAX_CHARS = 500
   URL_PLACEHOLDER_CHARS = 23
-  URL_PLACEHOLDER = "\1#{'x' * URL_PLACEHOLDER_CHARS}"
+  URL_PLACEHOLDER = 'x' * 23
 
   def validate(status)
     return unless status.local? && !status.reblog?
 
-    @status = status
-    status.errors.add(:text, I18n.t('statuses.over_character_limit', max: MAX_CHARS)) if too_long?
+    status.errors.add(:text, I18n.t('statuses.over_character_limit', max: MAX_CHARS)) if too_long?(status)
   end
 
   private
 
-  def too_long?
-    countable_length > MAX_CHARS
+  def too_long?(status)
+    countable_length(combined_text(status)) > MAX_CHARS
   end
 
-  def countable_length
-    total_text.mb_chars.grapheme_length
+  def countable_length(str)
+    str.mb_chars.grapheme_length
   end
 
-  def total_text
-    [@status.spoiler_text, countable_text].join
+  def combined_text(status)
+    [status.spoiler_text, countable_text(status.text)].join
   end
 
-  def countable_text
-    return '' if @status.text.nil?
+  def countable_text(str)
+    return '' if str.blank?
 
-    @status.text.dup.tap do |new_text|
-      new_text.gsub!(FetchLinkCardService::URL_PATTERN, URL_PLACEHOLDER)
-      new_text.gsub!(Account::MENTION_RE, '@\2')
+    # To ensure that we only give length concessions to entities that
+    # will be correctly parsed during formatting, we go through full
+    # entity extraction
+
+    entities = Extractor.remove_overlapping_entities(Extractor.extract_urls_with_indices(str, extract_url_without_protocol: false) + Extractor.extract_mentions_or_lists_with_indices(str))
+
+    rewrite_entities(str, entities) do |entity|
+      if entity[:url]
+        URL_PLACEHOLDER
+      elsif entity[:screen_name]
+        "@#{entity[:screen_name].split('@').first}"
+      end
     end
   end
+
+  def rewrite_entities(str, entities)
+    entities.sort_by! { |entity| entity[:indices].first }
+    result = ''.dup
+
+    last_index = entities.reduce(0) do |index, entity|
+      result << str[index...entity[:indices].first]
+      result << yield(entity)
+      entity[:indices].last
+    end
+
+    result << str[last_index..-1]
+    result
+  end
 end

app/views/admin/reports/show.html.haml

@@ -92,7 +92,7 @@
 %hr.spacer
 
 .speech-bubble
-  .speech-bubble__bubble= simple_format(@report.comment.presence || t('admin.reports.comment.none'))
+  .speech-bubble__bubble= simple_format(h(@report.comment.presence || t('admin.reports.comment.none')))
   .speech-bubble__owner
     - if @report.account.local?
       = admin_account_link_to @report.account

app/workers/activitypub/delivery_worker.rb

@@ -44,11 +44,7 @@ class ActivityPub::DeliveryWorker
   end
 
   def synchronization_header
-    "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
-  end
-
-  def inbox_url_prefix
-    @inbox_url[/http(s?):\/\/[^\/]+\//]
+    "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(@inbox_url)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
   end
 
   def perform_request

chart/values.yaml

@@ -8,7 +8,7 @@ image:
   # built from the most recent commit
   #
   # tag: latest
-  tag: v3.3.0
+  tag: v3.4.6
   # use `Always` when using `latest` tag
   pullPolicy: IfNotPresent
 

config/brakeman.ignore

@@ -60,46 +60,6 @@
       "confidence": "High",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "6e4051854bb62e2ddbc671f82d6c2328892e1134b8b28105ecba9b0122540714",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 479,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"          WITH first_degree AS (\\n            SELECT target_account_id\\n            FROM follows\\n            WHERE account_id = ?\\n            UNION ALL\\n            SELECT ?\\n          )\\n          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n          WHERE accounts.id IN (SELECT * FROM first_degree)\\n            AND #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Account",
-        "method": "advanced_search_for"
-      },
-      "user_input": "textsearch",
-      "confidence": "Medium",
-      "note": ""
-    },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "6f075c1484908e3ec9bed21ab7cf3c7866be8da3881485d1c82e13093aefcbd7",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/status.rb",
-      "line": 105,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "result.joins(\"LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Status",
-        "method": null
-      },
-      "user_input": "id",
-      "confidence": "Weak",
-      "note": ""
-    },
     {
       "warning_type": "Mass Assignment",
       "warning_code": 105,
@@ -140,26 +100,6 @@
       "confidence": "High",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "9251d682c4e2840e1b2fea91e7d758efe2097ecb7f6255c065e3750d25eb178c",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 448,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"        SELECT\\n          accounts.*,\\n          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n        FROM accounts\\n        WHERE #{query} @@ #{textsearch}\\n          AND accounts.suspended_at IS NULL\\n          AND accounts.moved_to_account_id IS NULL\\n        ORDER BY rank DESC\\n        LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Account",
-        "method": "search_for"
-      },
-      "user_input": "textsearch",
-      "confidence": "Medium",
-      "note": ""
-    },
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
@@ -220,26 +160,6 @@
       "confidence": "High",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "e21d8fee7a5805761679877ca35ed1029c64c45ef3b4012a30262623e1ba8bb9",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 495,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n          WHERE #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Account",
-        "method": "advanced_search_for"
-      },
-      "user_input": "textsearch",
-      "confidence": "Medium",
-      "note": ""
-    },
     {
       "warning_type": "Mass Assignment",
       "warning_code": 105,

config/environments/production.rb

@@ -118,7 +118,7 @@ Rails.application.configure do
     'Server'                 => 'Mastodon',
     'X-Frame-Options'        => 'DENY',
     'X-Content-Type-Options' => 'nosniff',
-    'X-XSS-Protection'       => '1; mode=block',
+    'X-XSS-Protection'       => '0',
     'Permissions-Policy'     => 'interest-cohort=()',
   }
 

db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb

@@ -1,6 +1,46 @@
 class RemoveFauxRemoteAccountDuplicates < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
+  class StreamEntry < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :stream_entries
+  end
+
+  class Status < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :statuses
+    has_many :favourites, inverse_of: :status, dependent: :destroy
+    has_many :mentions, dependent: :destroy, inverse_of: :status
+  end
+
+  class Favourite < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :favourites
+    belongs_to :status,  inverse_of: :favourites
+  end
+
+  class Mention < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :mentions
+    belongs_to :status
+  end
+
+  class Notification < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, optional: true
+    belongs_to :from_account, class_name: 'Account', optional: true
+    belongs_to :activity, polymorphic: true, optional: true
+  end
+
+  class Account < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    has_many :stream_entries, inverse_of: :account, dependent: :destroy
+    has_many :statuses, inverse_of: :account, dependent: :destroy
+    has_many :favourites, inverse_of: :account, dependent: :destroy
+    has_many :mentions, inverse_of: :account, dependent: :destroy
+    has_many :notifications, inverse_of: :account, dependent: :destroy
+  end
+
   def up
     local_domain = Rails.configuration.x.local_domain
 

db/migrate/20190715164535_add_instance_actor.rb

@@ -1,4 +1,9 @@
 class AddInstanceActor < ActiveRecord::Migration[5.2]
+  class Account < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    validates :username, uniqueness: { scope: :domain, case_sensitive: false }
+  end
+
   def up
     Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
   end

db/migrate/20191007013357_update_pt_locales.rb

@@ -1,4 +1,8 @@
 class UpdatePtLocales < ActiveRecord::Migration[5.2]
+  class User < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+  end
+
   disable_ddl_transaction!
 
   def up

db/views/follow_recommendations_v01.sql

@@ -20,7 +20,7 @@ FROM (
   HAVING count(follows.id) >= 5
   UNION ALL
   SELECT accounts.id AS account_id,
-         sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank,
+         sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank,
          'most_interactions' AS reason
   FROM status_stats
   INNER JOIN statuses ON statuses.id = status_stats.status_id
@@ -32,7 +32,7 @@ FROM (
     AND accounts.locked = 'f'
     AND accounts.discoverable = 't'
   GROUP BY accounts.id
-  HAVING sum(reblogs_count + favourites_count) >= 5
+  HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5
 ) t0
 GROUP BY account_id
 ORDER BY rank DESC

db/views/follow_recommendations_v02.sql

@@ -18,7 +18,7 @@ FROM (
   HAVING count(follows.id) >= 5
   UNION ALL
   SELECT account_summaries.account_id AS account_id,
-         sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank,
+         sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank,
          'most_interactions' AS reason
   FROM status_stats
   INNER JOIN statuses ON statuses.id = status_stats.status_id
@@ -28,7 +28,7 @@ FROM (
     AND account_summaries.sensitive = 'f'
     AND follow_recommendation_suppressions.id IS NULL
   GROUP BY account_summaries.account_id
-  HAVING sum(reblogs_count + favourites_count) >= 5
+  HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5
 ) t0
 GROUP BY account_id
 ORDER BY rank DESC

docker-compose.yml

@@ -43,7 +43,7 @@ services:
 
   web:
     build: .
-    image: tootsuite/mastodon
+    image: tootsuite/mastodon:v3.4.6
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -63,7 +63,7 @@ services:
 
   streaming:
     build: .
-    image: tootsuite/mastodon
+    image: tootsuite/mastodon:v3.4.6
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -80,7 +80,7 @@ services:
 
   sidekiq:
     build: .
-    image: tootsuite/mastodon
+    image: tootsuite/mastodon:v3.4.6
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/mastodon/migration_helpers.rb

@@ -295,7 +295,7 @@ module Mastodon
       table = Arel::Table.new(table_name)
 
       total = estimate_rows_in_table(table_name).to_i
-      if total == 0
+      if total < 1
         count_arel = table.project(Arel.star.count.as('count'))
         count_arel = yield table, count_arel if block_given?
 

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      3
+      7
     end
 
     def flags

lib/paperclip/response_with_limit_adapter.rb

@@ -17,9 +17,9 @@ module Paperclip
 
     def cache_current_values
       @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data'
-      @size = @target.response.content_length
       @tempfile = copy_to_tempfile(@target)
       @content_type = ContentTypeDetector.new(@tempfile.path).detect
+      @size = File.size(@tempfile)
     end
 
     def copy_to_tempfile(source)

lib/tasks/mastodon.rake

@@ -350,11 +350,11 @@ namespace :mastodon do
           end
         end.join("\n")
 
-        generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n"
+        generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup
 
         if incompatible_syntax
-          generated_header << "Some variables in this file will be interpreted differently whether you are\n"
-          generated_header << "using docker-compose or not.\n\n"
+          generated_header << "# Some variables in this file will be interpreted differently whether you are\n"
+          generated_header << "# using docker-compose or not.\n\n"
         end
 
         File.write(Rails.root.join('.env.production'), "#{generated_header}#{env_contents}\n")
@@ -441,7 +441,7 @@ namespace :mastodon do
 
   namespace :webpush do
     desc 'Generate VAPID key'
-    task generate_vapid_key: :environment do
+    task :generate_vapid_key do
       vapid_key = Webpush.generate_key
       puts "VAPID_PRIVATE_KEY=#{vapid_key.private_key}"
       puts "VAPID_PUBLIC_KEY=#{vapid_key.public_key}"

lib/tasks/tests.rake

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+namespace :tests do
+  namespace :migrations do
+    desc 'Populate the database with test data for 2.0.0'
+    task populate_v2: :environment do
+      admin_key   = OpenSSL::PKey::RSA.new(2048)
+      user_key    = OpenSSL::PKey::RSA.new(2048)
+      remote_key  = OpenSSL::PKey::RSA.new(2048)
+      remote_key2 = OpenSSL::PKey::RSA.new(2048)
+      remote_key3 = OpenSSL::PKey::RSA.new(2048)
+      admin_private_key    = ActiveRecord::Base.connection.quote(admin_key.to_pem)
+      admin_public_key     = ActiveRecord::Base.connection.quote(admin_key.public_key.to_pem)
+      user_private_key     = ActiveRecord::Base.connection.quote(user_key.to_pem)
+      user_public_key      = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem)
+      remote_public_key    = ActiveRecord::Base.connection.quote(remote_key.public_key.to_pem)
+      remote_public_key2   = ActiveRecord::Base.connection.quote(remote_key2.public_key.to_pem)
+      remote_public_key_ap = ActiveRecord::Base.connection.quote(remote_key3.public_key.to_pem)
+      local_domain = ActiveRecord::Base.connection.quote(Rails.configuration.x.local_domain)
+
+      ActiveRecord::Base.connection.execute(<<~SQL)
+        -- accounts
+
+        INSERT INTO "accounts"
+          (id, username, domain, private_key, public_key, created_at, updated_at)
+        VALUES
+          (1, 'admin', NULL, #{admin_private_key}, #{admin_public_key}, now(), now()),
+          (2, 'user',  NULL, #{user_private_key},  #{user_public_key},  now(), now());
+
+        INSERT INTO "accounts"
+          (id, username, domain, private_key, public_key, created_at, updated_at, remote_url, salmon_url)
+        VALUES
+          (3, 'remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
+           'https://remote.com/@remote', 'https://remote.com/salmon/1'),
+          (4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
+           'https://remote.com/@Remote', 'https://remote.com/salmon/1'),
+          (5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now(), now(),
+           'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1');
+
+        INSERT INTO "accounts"
+          (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url)
+        VALUES
+          (6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(),
+           1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers');
+
+        INSERT INTO "accounts"
+          (id, username, domain, private_key, public_key, created_at, updated_at)
+        VALUES
+          (7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()),
+          (8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now());
+
+        -- users
+
+        INSERT INTO "users"
+          (id, account_id, email, created_at, updated_at, admin)
+        VALUES
+          (1, 1, 'admin@localhost', now(), now(), true),
+          (2, 2, 'user@localhost', now(), now(), false);
+
+        INSERT INTO "users"
+          (id, account_id, email, created_at, updated_at, admin, locale)
+        VALUES
+          (3, 7, 'ptuser@localhost', now(), now(), false, 'pt');
+
+        -- statuses
+
+        INSERT INTO "statuses"
+          (id, account_id, text, created_at, updated_at)
+        VALUES
+          (1, 1, 'test', now(), now()),
+          (2, 1, '@remote@remote.com hello', now(), now()),
+          (3, 1, '@Remote@remote.com hello', now(), now()),
+          (4, 1, '@REMOTE@remote.com hello', now(), now());
+
+        INSERT INTO "statuses"
+          (id, account_id, text, created_at, updated_at, uri, local)
+        VALUES
+          (5, 1, 'activitypub status', now(), now(), 'https://localhost/users/admin/statuses/4', true);
+
+        INSERT INTO "statuses"
+          (id, account_id, text, created_at, updated_at)
+        VALUES
+          (6, 3, 'test', now(), now());
+
+        INSERT INTO "statuses"
+          (id, account_id, text, created_at, updated_at, in_reply_to_id, in_reply_to_account_id)
+        VALUES
+          (7, 4, '@admin hello', now(), now(), 3, 1);
+
+        INSERT INTO "statuses"
+          (id, account_id, text, created_at, updated_at)
+        VALUES
+          (8, 5, 'test', now(), now());
+
+        INSERT INTO "statuses"
+          (id, account_id, reblog_of_id, created_at, updated_at)
+        VALUES
+          (9, 1, 2, now(), now());
+
+        -- mentions (from previous statuses)
+
+        INSERT INTO "mentions"
+          (status_id, account_id, created_at, updated_at)
+        VALUES
+          (2, 3, now(), now()),
+          (3, 4, now(), now()),
+          (4, 5, now(), now());
+
+        -- stream entries
+
+        INSERT INTO "stream_entries"
+          (activity_id, account_id, activity_type, created_at, updated_at)
+        VALUES
+          (1, 1, 'status', now(), now()),
+          (2, 1, 'status', now(), now()),
+          (3, 1, 'status', now(), now()),
+          (4, 1, 'status', now(), now()),
+          (5, 1, 'status', now(), now()),
+          (6, 3, 'status', now(), now()),
+          (7, 4, 'status', now(), now()),
+          (8, 5, 'status', now(), now()),
+          (9, 1, 'status', now(), now());
+
+
+        -- custom emoji
+
+        INSERT INTO "custom_emojis"
+          (shortcode, created_at, updated_at)
+        VALUES
+          ('test', now(), now()),
+          ('Test', now(), now()),
+          ('blobcat', now(), now());
+
+        INSERT INTO "custom_emojis"
+          (shortcode, domain, uri, created_at, updated_at)
+        VALUES
+          ('blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
+          ('blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
+          ('Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now());
+
+        -- favourites
+
+        INSERT INTO "favourites"
+          (account_id, status_id, created_at, updated_at)
+        VALUES
+          (1, 1, now(), now()),
+          (1, 7, now(), now()),
+          (4, 1, now(), now()),
+          (3, 1, now(), now()),
+          (5, 1, now(), now());
+
+        -- pinned statuses
+
+        INSERT INTO "status_pins"
+          (account_id, status_id, created_at, updated_at)
+        VALUES
+          (1, 1, now(), now()),
+          (3, 6, now(), now()),
+          (4, 7, now(), now());
+
+        -- follows
+
+        INSERT INTO "follows"
+          (account_id, target_account_id, created_at, updated_at)
+        VALUES
+          (1, 5, now(), now()),
+          (6, 2, now(), now()),
+          (5, 2, now(), now()),
+          (6, 1, now(), now());
+
+        -- follow requests
+
+        INSERT INTO "follow_requests"
+          (account_id, target_account_id, created_at, updated_at)
+        VALUES
+          (2, 5, now(), now()),
+          (5, 1, now(), now());
+      SQL
+    end
+  end
+end

spec/helpers/jsonld_helper_spec.rb

@@ -89,4 +89,86 @@ describe JsonLdHelper do
       expect(fetch_resource_without_id_validation('https://host.test/')).to eq({})
     end
   end
+
+  context 'compaction and forwarding' do
+    let(:json) do
+      {
+        '@context' => [
+          'https://www.w3.org/ns/activitystreams',
+          'https://w3id.org/security/v1',
+          {
+            'obsolete' => 'http://ostatus.org#',
+            'convo' => 'obsolete:conversation',
+            'new' => 'https://obscure-unreleased-test.joinmastodon.org/#',
+          },
+        ],
+        'type' => 'Create',
+        'to' => ['https://www.w3.org/ns/activitystreams#Public'],
+        'object' => {
+          'id' => 'https://example.com/status',
+          'type' => 'Note',
+          'inReplyTo' => nil,
+          'convo' => 'https://example.com/conversation',
+          'tag' => [
+            {
+              'type' => 'Mention',
+              'href' => ['foo'],
+            }
+          ],
+        },
+        'signature' => {
+          'type' => 'RsaSignature2017',
+          'created' => '2022-02-02T12:00:00Z',
+          'creator' => 'https://example.com/actor#main-key',
+          'signatureValue' => 'some-sig',
+        },
+      }
+    end
+
+    describe '#compact' do
+      it 'properly compacts JSON-LD with alternative context definitions' do
+        expect(compact(json).dig('object', 'conversation')).to eq 'https://example.com/conversation'
+      end
+
+      it 'compacts single-item arrays' do
+        expect(compact(json).dig('object', 'tag', 'href')).to eq 'foo'
+      end
+
+      it 'compacts the activistreams Public collection' do
+        expect(compact(json)['to']).to eq 'as:Public'
+      end
+
+      it 'properly copies signature' do
+        expect(compact(json)['signature']).to eq json['signature']
+      end
+    end
+
+    describe 'patch_for_forwarding!' do
+      it 'properly patches incompatibilities' do
+        json['object'].delete('convo')
+        compacted = compact(json)
+        patch_for_forwarding!(json, compacted)
+        expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public']
+        expect(compacted.dig('object', 'tag', 0, 'href')).to eq ['foo']
+        expect(safe_for_forwarding?(json, compacted)).to eq true
+      end
+    end
+
+    describe 'safe_for_forwarding?' do
+      it 'deems a safe compacting as such' do
+        json['object'].delete('convo')
+        compacted = compact(json)
+        deemed_compatible = patch_for_forwarding!(json, compacted)
+        expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public']
+        expect(safe_for_forwarding?(json, compacted)).to eq true
+      end
+
+      it 'deems an unsafe compacting as such' do
+        compacted = compact(json)
+        deemed_compatible = patch_for_forwarding!(json, compacted)
+        expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public']
+        expect(safe_for_forwarding?(json, compacted)).to eq false
+      end
+    end
+  end
 end

spec/models/account_spec.rb

@@ -5,6 +5,37 @@ RSpec.describe Account, type: :model do
     let(:bob) { Fabricate(:account, username: 'bob') }
     subject { Fabricate(:account) }
 
+    describe '#suspend!' do
+      it 'marks the account as suspended' do
+        subject.suspend!
+        expect(subject.suspended?).to be true
+      end
+
+      it 'creates a deletion request' do
+        subject.suspend!
+        expect(AccountDeletionRequest.where(account: subject).exists?).to be true
+      end
+
+      context 'when the account is of a local user' do
+        let!(:subject) { Fabricate(:account, user: Fabricate(:user, email: 'foo+bar@domain.org')) }
+
+        it 'creates a canonical domain block' do
+          subject.suspend!
+          expect(CanonicalEmailBlock.block?(subject.user_email)).to be true
+        end
+
+        context 'when a canonical domain block already exists for that email' do
+          before do
+            Fabricate(:canonical_email_block, email: subject.user_email)
+          end
+
+          it 'does not raise an error' do
+            expect { subject.suspend! }.not_to raise_error
+          end
+        end
+      end
+    end
+
     describe '#follow!' do
       it 'creates a follow' do
         follow = subject.follow!(bob)

spec/models/concerns/account_interactions_spec.rb

@@ -539,46 +539,57 @@ describe AccountInteractions do
     end
   end
 
-  describe '#followers_hash' do
+  describe '#remote_followers_hash' do
     let(:me) { Fabricate(:account, username: 'Me') }
     let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') }
     let(:remote_2) { Fabricate(:account, username: 'bob', domain: 'example.org', uri: 'https://example.org/users/bob') }
-    let(:remote_3) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') }
+    let(:remote_3) { Fabricate(:account, username: 'instance-actor', domain: 'example.org', uri: 'https://example.org') }
+    let(:remote_4) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') }
 
     before do
       remote_1.follow!(me)
       remote_2.follow!(me)
       remote_3.follow!(me)
+      remote_4.follow!(me)
       me.follow!(remote_1)
     end
 
-    context 'on a local user' do
-      it 'returns correct hash for remote domains' do
-        expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
-        expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e'
-      end
-
-      it 'invalidates cache as needed when removing or adding followers' do
-        expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
-        remote_1.unfollow!(me)
-        expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff'
-        remote_1.follow!(me)
-        expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
-      end
+    it 'returns correct hash for remote domains' do
+      expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7'
+      expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e'
+      expect(me.remote_followers_hash('https://foo.org.evil.com/')).to eq '0000000000000000000000000000000000000000000000000000000000000000'
+      expect(me.remote_followers_hash('https://foo')).to eq '0000000000000000000000000000000000000000000000000000000000000000'
     end
 
-    context 'on a remote user' do
-      it 'returns correct hash for remote domains' do
-        expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
-      end
+    it 'invalidates cache as needed when removing or adding followers' do
+      expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7'
+      remote_3.unfollow!(me)
+      expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
+      remote_1.unfollow!(me)
+      expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff'
+      remote_1.follow!(me)
+      expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec'
+    end
+  end
 
-      it 'invalidates cache as needed when removing or adding followers' do
-        expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
-        me.unfollow!(remote_1)
-        expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000'
-        me.follow!(remote_1)
-        expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
-      end
+  describe '#local_followers_hash' do
+    let(:me) { Fabricate(:account, username: 'Me') }
+    let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') }
+
+    before do
+      me.follow!(remote_1)
+    end
+
+    it 'returns correct hash for local users' do
+      expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
+    end
+
+    it 'invalidates cache as needed when removing or adding followers' do
+      expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
+      me.unfollow!(remote_1)
+      expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000'
+      me.follow!(remote_1)
+      expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me))
     end
   end
 

spec/models/status_spec.rb

@@ -267,6 +267,87 @@ RSpec.describe Status, type: :model do
     end
   end
 
+  describe '.tagged_with' do
+    let(:tag1)     { Fabricate(:tag) }
+    let(:tag2)     { Fabricate(:tag) }
+    let(:tag3)     { Fabricate(:tag) }
+    let!(:status1) { Fabricate(:status, tags: [tag1]) }
+    let!(:status2) { Fabricate(:status, tags: [tag2]) }
+    let!(:status3) { Fabricate(:status, tags: [tag3]) }
+    let!(:status4) { Fabricate(:status, tags: []) }
+    let!(:status5) { Fabricate(:status, tags: [tag1, tag2, tag3]) }
+
+    context 'when given one tag' do
+      it 'returns the expected statuses' do
+        expect(Status.tagged_with([tag1.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status5.id]
+        expect(Status.tagged_with([tag2.id]).reorder(:id).pluck(:id).uniq).to eq [status2.id, status5.id]
+        expect(Status.tagged_with([tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status3.id, status5.id]
+      end
+    end
+
+    context 'when given multiple tags' do
+      it 'returns the expected statuses' do
+        expect(Status.tagged_with([tag1.id, tag2.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status2.id, status5.id]
+        expect(Status.tagged_with([tag1.id, tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status3.id, status5.id]
+        expect(Status.tagged_with([tag2.id, tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status2.id, status3.id, status5.id]
+      end
+    end
+  end
+
+  describe '.tagged_with_all' do
+    let(:tag1)     { Fabricate(:tag) }
+    let(:tag2)     { Fabricate(:tag) }
+    let(:tag3)     { Fabricate(:tag) }
+    let!(:status1) { Fabricate(:status, tags: [tag1]) }
+    let!(:status2) { Fabricate(:status, tags: [tag2]) }
+    let!(:status3) { Fabricate(:status, tags: [tag3]) }
+    let!(:status4) { Fabricate(:status, tags: []) }
+    let!(:status5) { Fabricate(:status, tags: [tag1, tag2]) }
+
+    context 'when given one tag' do
+      it 'returns the expected statuses' do
+        expect(Status.tagged_with_all([tag1.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status5.id]
+        expect(Status.tagged_with_all([tag2.id]).reorder(:id).pluck(:id).uniq).to eq [status2.id, status5.id]
+        expect(Status.tagged_with_all([tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status3.id]
+      end
+    end
+
+    context 'when given multiple tags' do
+      it 'returns the expected statuses' do
+        expect(Status.tagged_with_all([tag1.id, tag2.id]).reorder(:id).pluck(:id).uniq).to eq [status5.id]
+        expect(Status.tagged_with_all([tag1.id, tag3.id]).reorder(:id).pluck(:id).uniq).to eq []
+        expect(Status.tagged_with_all([tag2.id, tag3.id]).reorder(:id).pluck(:id).uniq).to eq []
+      end
+    end
+  end
+
+  describe '.tagged_with_none' do
+    let(:tag1)     { Fabricate(:tag) }
+    let(:tag2)     { Fabricate(:tag) }
+    let(:tag3)     { Fabricate(:tag) }
+    let!(:status1) { Fabricate(:status, tags: [tag1]) }
+    let!(:status2) { Fabricate(:status, tags: [tag2]) }
+    let!(:status3) { Fabricate(:status, tags: [tag3]) }
+    let!(:status4) { Fabricate(:status, tags: []) }
+    let!(:status5) { Fabricate(:status, tags: [tag1, tag2, tag3]) }
+
+    context 'when given one tag' do
+      it 'returns the expected statuses' do
+        expect(Status.tagged_with_none([tag1.id]).reorder(:id).pluck(:id).uniq).to eq [status2.id, status3.id, status4.id]
+        expect(Status.tagged_with_none([tag2.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status3.id, status4.id]
+        expect(Status.tagged_with_none([tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status2.id, status4.id]
+      end
+    end
+
+    context 'when given multiple tags' do
+      it 'returns the expected statuses' do
+        expect(Status.tagged_with_none([tag1.id, tag2.id]).reorder(:id).pluck(:id).uniq).to eq [status3.id, status4.id]
+        expect(Status.tagged_with_none([tag1.id, tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status2.id, status4.id]
+        expect(Status.tagged_with_none([tag2.id, tag3.id]).reorder(:id).pluck(:id).uniq).to eq [status1.id, status4.id]
+      end
+    end
+  end
+
   describe '.permitted_for' do
     subject { described_class.permitted_for(target_account, account).pluck(:visibility) }
 

spec/services/notify_service_spec.rb

@@ -64,8 +64,9 @@ RSpec.describe NotifyService, type: :service do
         is_expected.to_not change(Notification, :count)
       end
 
-      context 'if the message chain initiated by recipient, but is not direct message' do
+      context 'if the message chain is initiated by recipient, but is not direct message' do
         let(:reply_to) { Fabricate(:status, account: recipient) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
         it 'does not notify' do
@@ -73,8 +74,20 @@ RSpec.describe NotifyService, type: :service do
         end
       end
 
-      context 'if the message chain initiated by recipient and is direct message' do
+      context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do
+        let(:reply_to) { Fabricate(:status, account: recipient) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
+        let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) }
+        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) }
+
+        it 'does not notify' do
+          is_expected.to_not change(Notification, :count)
+        end
+      end
+
+      context 'if the message chain is initiated by the recipient with a mention to the sender' do
         let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) }
+        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
         let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) }
 
         it 'does notify' do

spec/validators/status_length_validator_spec.rb

@@ -50,6 +50,13 @@ describe StatusLengthValidator do
       expect(status.errors).to have_received(:add)
     end
 
+    it 'does not count overly long URLs as 23 characters flat' do
+      text = "http://example.com/valid?#{'#foo?' * 1000}"
+      status = double(spoiler_text: '', text: text, errors: double(add: nil), local?: true, reblog?: false)
+      subject.validate(status)
+      expect(status.errors).to have_received(:add)
+    end
+
     it 'counts only the front part of remote usernames' do
       text   = ('a' * 475) + " @alice@#{'b' * 30}.com"
       status = double(spoiler_text: '', text: text, errors: double(add: nil), local?: true, reblog?: false)
@@ -57,5 +64,13 @@ describe StatusLengthValidator do
       subject.validate(status)
       expect(status.errors).to_not have_received(:add)
     end
+
+    it 'does count both parts of remote usernames for overly long domains' do
+      text   = "@alice@#{'b' * 500}.com"
+      status = double(spoiler_text: '', text: text, errors: double(add: nil), local?: true, reblog?: false)
+
+      subject.validate(status)
+      expect(status.errors).to have_received(:add)
+    end
   end
 end

spec/workers/activitypub/delivery_worker_spec.rb

@@ -11,7 +11,7 @@ describe ActivityPub::DeliveryWorker do
   let(:payload) { 'test' }
 
   before do
-    allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/').and_return('somehash')
+    allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/api').and_return('somehash')
   end
 
   describe 'perform' do