mstdn.y-zu.org/mastodon
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge tag 'v4.1.9' into features/4.1.9

Browse Source
This commit is contained in:
YoheiZuho 2023-10-11 18:13:24 +09:00
commit 70cf68fc6e
45 changed files with 628 additions and 498 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
.circleci/config.yml
View File

@ -1,225 +0,0 @@
version: 2.1
orbs:
ruby: circleci/ruby@2.0.0
node: circleci/node@5.0.3
executors:
default:
parameters:
ruby-version:
type: string
docker:
- image: cimg/ruby:<< parameters.ruby-version >>
environment:
BUNDLE_JOBS: 3
BUNDLE_RETRY: 3
CONTINUOUS_INTEGRATION: true
DB_HOST: localhost
DB_USER: root
DISABLE_SIMPLECOV: true
RAILS_ENV: test
- image: cimg/postgres:14.5
environment:
POSTGRES_USER: root
POSTGRES_HOST_AUTH_METHOD: trust
- image: cimg/redis:7.0
commands:
install-system-dependencies:
steps:
- run:
name: Install system dependencies
command: |
sudo apt-get update
sudo apt-get install -y libicu-dev libidn11-dev
install-ruby-dependencies:
parameters:
ruby-version:
type: string
steps:
- run:
command: |
bundle config clean 'true'
bundle config frozen 'true'
bundle config without 'development production'
name: Set bundler settings
- ruby/install-deps:
bundler-version: '2.3.26'
key: ruby<< parameters.ruby-version >>-gems-v1
wait-db:
steps:
- run:
command: dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m
name: Wait for PostgreSQL and Redis
jobs:
build:
docker:
- image: cimg/ruby:3.0-node
environment:
RAILS_ENV: test
steps:
- checkout
- install-system-dependencies
- install-ruby-dependencies:
ruby-version: '3.0'
- node/install-packages:
cache-version: v1
pkg-manager: yarn
- run:
command: |
export NODE_OPTIONS=--openssl-legacy-provider
./bin/rails assets:precompile
name: Precompile assets
- persist_to_workspace:
paths:
- public/assets
- public/packs-test
root: .
test:
parameters:
ruby-version:
type: string
executor:
name: default
ruby-version: << parameters.ruby-version >>
environment:
ALLOW_NOPAM: true
PAM_ENABLED: true
PAM_DEFAULT_SERVICE: pam_test
PAM_CONTROLLED_SERVICE: pam_test_controlled
parallelism: 4
steps:
- checkout
- install-system-dependencies
- run:
command: sudo apt-get install -y ffmpeg imagemagick libpam-dev
name: Install additional system dependencies
- run:
command: bundle config with 'pam_authentication'
name: Enable PAM authentication
- install-ruby-dependencies:
ruby-version: << parameters.ruby-version >>
- attach_workspace:
at: .
- wait-db
- run:
command: ./bin/rails db:create db:schema:load db:seed
name: Load database schema
- ruby/rspec-test
test-migrations:
executor:
name: default
ruby-version: '3.0'
steps:
- checkout
- install-system-dependencies
- install-ruby-dependencies:
ruby-version: '3.0'
- wait-db
- run:
command: ./bin/rails db:create
name: Create database
- run:
command: ./bin/rails db:migrate VERSION=20171010025614
name: Run migrations up to v2.0.0
- run:
command: ./bin/rails tests:migrations:populate_v2
name: Populate database with test data
- run:
command: ./bin/rails db:migrate VERSION=20180514140000
name: Run migrations up to v2.4.0
- run:
command: ./bin/rails tests:migrations:populate_v2_4
name: Populate database with test data
- run:
command: ./bin/rails db:migrate VERSION=20180707154237
name: Run migrations up to v2.4.3
- run:
command: ./bin/rails tests:migrations:populate_v2_4_3
name: Populate database with test data
- run:
command: ./bin/rails db:migrate
name: Run all remaining migrations
- run:
command: ./bin/rails tests:migrations:check_database
name: Check migration result
test-two-step-migrations:
executor:
name: default
ruby-version: '3.0'
steps:
- checkout
- install-system-dependencies
- install-ruby-dependencies:
ruby-version: '3.0'
- wait-db
- run:
command: ./bin/rails db:create
name: Create database
- run:
command: ./bin/rails db:migrate VERSION=20171010025614
name: Run migrations up to v2.0.0
- run:
command: ./bin/rails tests:migrations:populate_v2
name: Populate database with test data
- run:
command: ./bin/rails db:migrate VERSION=20180514140000
name: Run pre-deployment migrations up to v2.4.0
environment:
SKIP_POST_DEPLOYMENT_MIGRATIONS: true
- run:
command: ./bin/rails tests:migrations:populate_v2_4
name: Populate database with test data
- run:
command: ./bin/rails db:migrate VERSION=20180707154237
name: Run migrations up to v2.4.3
environment:
SKIP_POST_DEPLOYMENT_MIGRATIONS: true
- run:
command: ./bin/rails tests:migrations:populate_v2_4_3
name: Populate database with test data
- run:
command: ./bin/rails db:migrate
name: Run all remaining pre-deployment migrations
environment:
SKIP_POST_DEPLOYMENT_MIGRATIONS: true
- run:
command: ./bin/rails db:migrate
name: Run all post-deployment migrations
- run:
command: ./bin/rails tests:migrations:check_database
name: Check migration result
workflows:
version: 2
build-and-test:
jobs:
- build
- test:
matrix:
parameters:
ruby-version:
- '2.7'
- '3.0'
name: test-ruby<< matrix.ruby-version >>
requires:
- build
- test-migrations:
requires:
- build
- test-two-step-migrations:
requires:
- build
- node/run:
cache-version: v1
name: test-webui
pkg-manager: yarn
requires:
- build
version: '16.18'
yarn-run: test:jest

92
.github/workflows/build-container-image.yml vendored Normal file
View File

@ -0,0 +1,92 @@
on:
workflow_call:
inputs:
platforms:
required: true
type: string
cache:
type: boolean
default: true
use_native_arm64_builder:
type: boolean
push_to_images:
type: string
flavor:
type: string
tags:
type: string
labels:
type: string
jobs:
build-image:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: docker/setup-qemu-action@v2
if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder
- uses: docker/setup-buildx-action@v2
id: buildx
if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }}
- name: Start a local Docker Builder
if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
run: |
docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234
- uses: docker/setup-buildx-action@v2
id: buildx-native
if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
with:
driver: remote
endpoint: tcp://localhost:1234
platforms: linux/amd64
append: |
- endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865
platforms: linux/arm64
name: mastodon-docker-builder-arm64-01
driver-opts:
- servername=mastodon-docker-builder-arm64-01
env:
BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }}
BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }}
BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }}
- name: Log in to Docker Hub
if: contains(inputs.push_to_images, 'tootsuite')
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to the Github Container registry
if: contains(inputs.push_to_images, 'ghcr.io')
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/metadata-action@v4
id: meta
if: ${{ inputs.push_to_images != '' }}
with:
images: ${{ inputs.push_to_images }}
flavor: ${{ inputs.flavor }}
tags: ${{ inputs.tags }}
labels: ${{ inputs.labels }}
- uses: docker/build-push-action@v4
with:
context: .
platforms: ${{ inputs.platforms }}
provenance: false
builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}
push: ${{ inputs.push_to_images != '' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: ${{ inputs.cache && 'type=gha' || '' }}
cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }}

70
.github/workflows/build-image.yml vendored
View File

@ -1,70 +0,0 @@
name: Build container image
on:
workflow_dispatch:
push:
branches:
- 'main'
tags:
- '*'
pull_request:
paths:
- .github/workflows/build-image.yml
- Dockerfile
permissions:
contents: read
packages: write
jobs:
build-image:
runs-on: ubuntu-latest
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
steps:
- uses: actions/checkout@v3
- uses: hadolint/hadolint-action@v3.1.0
- uses: docker/setup-qemu-action@v2
- uses: docker/setup-buildx-action@v2
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
- name: Log in to the Github Container registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
- uses: docker/metadata-action@v4
id: meta
with:
images: |
tootsuite/mastodon
ghcr.io/mastodon/mastodon
flavor: |
latest=auto
tags: |
type=edge,branch=main
type=pep440,pattern={{raw}}
type=pep440,pattern=v{{major}}.{{minor}}
type=ref,event=pr
- uses: docker/build-push-action@v4
with:
context: .
platforms: linux/amd64,linux/arm64
provenance: false
builder: ${{ steps.buildx.outputs.name }}
push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

29
.github/workflows/build-releases.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: Build container release images
on:
push:
tags:
- '*'
permissions:
contents: read
packages: write
jobs:
build-image:
uses: ./.github/workflows/build-container-image.yml
with:
platforms: linux/amd64,linux/arm64
use_native_arm64_builder: true
push_to_images: |
tootsuite/mastodon
ghcr.io/mastodon/mastodon
# Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
cache: false
# Only tag with latest when ran against the latest stable branch
# This needs to be updated after each minor version release
flavor: |
latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') }}
tags: |
type=pep440,pattern={{raw}}
type=pep440,pattern=v{{major}}.{{minor}}
secrets: inherit

41
.github/workflows/lint-ruby.yml vendored
View File

@ -1,41 +0,0 @@
name: Ruby Linting
on:
push:
branches-ignore:
- 'dependabot/**'
paths:
- 'Gemfile*'
- '.rubocop.yml'
- '**/*.rb'
- '**/*.rake'
- '.github/workflows/lint-ruby.yml'
pull_request:
paths:
- 'Gemfile*'
- '.rubocop.yml'
- '**/*.rb'
- '**/*.rake'
- '.github/workflows/lint-ruby.yml'
jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set-up RuboCop Problem Mathcher
uses: r7kamura/rubocop-problem-matchers-action@v1
- name: Run rubocop
uses: github/super-linter@v4
env:
DEFAULT_BRANCH: main
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LINTER_RULES_PATH: .
RUBY_CONFIG_FILE: .rubocop.yml
VALIDATE_ALL_CODEBASE: false
VALIDATE_RUBY: true

15
.github/workflows/test-image-build.yml vendored Normal file
View File

@ -0,0 +1,15 @@
name: Test container image build
on:
pull_request:
permissions:
contents: read
jobs:
build-image:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
uses: ./.github/workflows/build-container-image.yml
with:
platforms: linux/amd64 # Testing only on native platform so it is performant

62
CHANGELOG.md
View File

@ -3,6 +3,68 @@ Changelog
All notable changes to this project will be documented in this file.
## [4.1.9] - 2023-09-20
### Fixed
- Fix post translation erroring out ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26990))
## [4.1.8] - 2023-09-19
### Fixed
- Fix post edits not being forwarded as expected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26936))
- Fix moderator rights inconsistencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26729))
- Fix crash when encountering invalid URL ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26814))
- Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409))
- Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500))
- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608))
### Security
- Fix missing HTML sanitization in translation API (CVE-2023-42452)
- Fix incorrect domain name normalization (CVE-2023-42451)
## [4.1.7] - 2023-09-05
### Changed
- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028))
### Fixed
- **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392))
- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237))
- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727))
## [4.1.6] - 2023-07-31
### Fixed
- Fix memory leak in streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26228))
- Fix wrong filters sometimes applying in streaming ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26159), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26213), [renchap](https://github.com/mastodon/mastodon/pull/26233))
- Fix incorrect connect timeout in outgoing requests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26116))
## [4.1.5] - 2023-07-21
### Added
- Add check preventing Sidekiq workers from running with Makara configured ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25850))
### Changed
- Change request timeout handling to use a longer deadline ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26055))
### Fixed
- Fix moderation interface for remote instances with a .zip TLD ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
- Fix remote accounts being possibly persisted to database with incomplete protocol values ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886))
- Fix trending publishers table not rendering correctly on narrow screens ([vmstan](https://github.com/mastodon/mastodon/pull/25945))
### Security
- Fix CSP headers being unintentionally wide ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26105))
## [4.1.4] - 2023-07-07
### Fixed

1
Dockerfile
View File

@ -17,6 +17,7 @@ COPY Gemfile* package.json yarn.lock /opt/mastodon/
# hadolint ignore=DL3008
RUN apt-get update && \
apt-get -yq dist-upgrade && \
apt-get install -y --no-install-recommends build-essential \
ca-certificates \
git \

2
app/controllers/admin/domain_blocks_controller.rb
View File

@ -37,7 +37,7 @@ module Admin
@domain_block.errors.delete(:domain)
render :new
else
if existing_domain_block.present?
if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip)
@domain_block = existing_domain_block
@domain_block.update(resource_params)
end

5
app/controllers/api/v1/timelines/tag_controller.rb
View File

@ -1,6 +1,7 @@
# frozen_string_literal: true
class Api::V1::Timelines::TagController < Api::BaseController
before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth?
before_action :load_tag
after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
@ -15,6 +16,10 @@ class Api::V1::Timelines::TagController < Api::BaseController
private
def require_auth?
!Setting.timeline_preview
end
def load_tag
@tag = Tag.find_normalized(params[:id])
end

0
app/models/account_statuses_filter.rb → app/lib/account_statuses_filter.rb
View File

6
app/lib/activitypub/activity/flag.rb
View File

@ -16,7 +16,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
@account,
target_account,
status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id),
comment: @json['content'] || '',
comment: report_comment,
uri: report_uri
)
end
@ -35,4 +35,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
def report_uri
@json['id'] unless @json['id'].nil? || invalid_origin?(@json['id'])
end
def report_comment
(@json['content'] || '')[0...5000]
end
end

2
app/lib/activitypub/activity/update.rb
View File

@ -28,6 +28,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
return if @status.nil?
ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id])
ActivityPub::ProcessStatusUpdateService.new.call(@status, @json, @object, request_id: @options[:request_id])
end
end

4
app/lib/activitypub/tag_manager.rb
View File

@ -27,6 +27,8 @@ class ActivityPub::TagManager
when :note, :comment, :activity
return activity_account_status_url(target.account, target) if target.reblog?
short_account_status_url(target.account, target)
when :flag
target.uri
end
end
@ -41,6 +43,8 @@ class ActivityPub::TagManager
account_status_url(target.account, target)
when :emoji
emoji_url(target)
when :flag
target.uri
end
end

9
app/lib/admin/account_statuses_filter.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
class Admin::AccountStatusesFilter < AccountStatusesFilter
private
def blocked?
false
end
end

13
app/lib/plain_text_formatter.rb
View File

@ -1,9 +1,7 @@
# frozen_string_literal: true
class PlainTextFormatter
include ActionView::Helpers::TextHelper
NEWLINE_TAGS_RE = /(<br \/>|<br>|<\/p>)+/.freeze
NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
attr_reader :text, :local
@ -18,7 +16,10 @@ class PlainTextFormatter
if local?
text
else
html_entities.decode(strip_tags(insert_newlines)).chomp
node = Nokogiri::HTML.fragment(insert_newlines)
# Elements that are entirely removed with our Sanitize config
node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
node.text.chomp
end
end
@ -27,8 +28,4 @@ class PlainTextFormatter
def insert_newlines
text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
end
def html_entities
HTMLEntities.new
end
end

28
app/lib/request.rb
View File

@ -4,14 +4,22 @@ require 'ipaddr'
require 'socket'
require 'resolv'
# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
# Use our own timeout class to avoid using HTTP.rb's timeout block
# around the Socket#open method, since we use our own timeout blocks inside
# that method
#
# Also changes how the read timeout behaves so that it is cumulative (closer
# to HTTP::Timeout::Global, but still having distinct timeouts for other
# operation types)
class HTTP::Timeout::PerOperation
class PerOperationWithDeadline < HTTP::Timeout::PerOperation
READ_DEADLINE = 30
def initialize(*args)
super
@read_deadline = options.fetch(:read_deadline, READ_DEADLINE)
end
def connect(socket_class, host, port, nodelay = false)
@socket = socket_class.open(host, port)
@socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
@ -24,7 +32,7 @@ class HTTP::Timeout::PerOperation
# Read data from the socket
def readpartial(size, buffer = nil)
@deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout
@deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_deadline
timeout = false
loop do
@ -33,7 +41,8 @@ class HTTP::Timeout::PerOperation
return :eof if result.nil?
remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0
raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout
raise HTTP::TimeoutError, "Read timed out after a total of #{@read_deadline} seconds" if remaining_time <= 0
return result if result != :wait_readable
# marking the socket for timeout. Why is this not being raised immediately?
@ -46,7 +55,7 @@ class HTTP::Timeout::PerOperation
# timeout. Else, the first timeout was a proper timeout.
# This hack has to be done because io/wait#wait_readable doesn't provide a value for when
# the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
timeout = true unless @socket.to_io.wait_readable(remaining_time)
timeout = true unless @socket.to_io.wait_readable([remaining_time, @read_timeout].min)
end
end
end
@ -57,7 +66,7 @@ class Request
# We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
# and 5s timeout on the TLS handshake, meaning the worst case should take
# about 15s in total
TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze
include RoutingHelper
@ -69,6 +78,7 @@ class Request
@http_client = options.delete(:http_client)
@allow_local = options.delete(:allow_local)
@options = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
@options = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT)
@options = @options.merge(proxy_url) if use_proxy?
@headers = {}
@ -129,7 +139,7 @@ class Request
end
def http_client
HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3)
HTTP.use(:auto_inflate).follow(max_hops: 3)
end
end
@ -275,11 +285,11 @@ class Request
end
until socks.empty?
_, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
_, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect_timeout])
if available_socks.nil?
socks.each(&:close)
raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect_timeout]} seconds"
end
available_socks.each do |sock|

8
app/lib/tag_manager.rb
View File

@ -7,18 +7,18 @@ class TagManager
include RoutingHelper
def web_domain?(domain)
domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.web_domain).zero?
domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.web_domain).zero?
end
def local_domain?(domain)
domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero?
domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.local_domain).zero?
end
def normalize_domain(domain)
return if domain.nil?
uri = Addressable::URI.new
uri.host = domain.gsub(/[\/]/, '')
uri.host = domain.delete_suffix('/')
uri.normalized_host
end
@ -28,7 +28,7 @@ class TagManager
domain = uri.host + (uri.port ? ":#{uri.port}" : '')
TagManager.instance.web_domain?(domain)
rescue Addressable::URI::InvalidURIError
rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
false
end
end

2
app/lib/translation_service/deepl.rb
View File

@ -46,7 +46,7 @@ class TranslationService::DeepL < TranslationService
raise UnexpectedResponseError unless json.is_a?(Hash)
Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
Translation.new(text: Sanitize.fragment(json.dig('translations', 0, 'text'), Sanitize::Config::MASTODON_STRICT), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com')
rescue Oj::ParseError
raise UnexpectedResponseError
end

2
app/lib/translation_service/libre_translate.rb
View File

@ -37,7 +37,7 @@ class TranslationService::LibreTranslate < TranslationService
raise UnexpectedResponseError unless json.is_a?(Hash)
Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate')
Translation.new(text: Sanitize.fragment(json['translatedText'], Sanitize::Config::MASTODON_STRICT), detected_source_language: source_language, provider: 'LibreTranslate')
rescue Oj::ParseError
raise UnexpectedResponseError
end

3
app/lib/video_metadata_extractor.rb
View File

@ -43,6 +43,9 @@ class VideoMetadataExtractor
@height = video_stream[:height]
@frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
@r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
# For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we
# should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue.
@frame_rate ||= @r_frame_rate
end
if (audio_stream = audio_streams.first)

2
app/models/admin/status_batch_action.rb
View File

@ -140,6 +140,6 @@ class Admin::StatusBatchAction
end
def allowed_status_ids
AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
Admin::AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id)
end
end

9
app/models/report.rb
View File

@ -39,7 +39,10 @@ class Report < ApplicationRecord
scope :resolved, -> { where.not(action_taken_at: nil) }
scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) }
validates :comment, length: { maximum: 1_000 }
# A report is considered local if the reporter is local
delegate :local?, to: :account
validates :comment, length: { maximum: 1_000 }, if: :local?
validates :rule_ids, absence: true, unless: :violation?
validate :validate_rule_ids
@ -50,10 +53,6 @@ class Report < ApplicationRecord
violation: 2_000,
}
def local?
false # Force uri_for to use uri attribute
end
before_validation :set_uri, only: :create
after_create_commit :trigger_webhooks

12
app/models/status.rb
View File

@ -371,13 +371,25 @@ class Status < ApplicationRecord
account_ids.uniq!
status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
return if account_ids.empty?
accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
cached_items.each do |item|
item.account = accounts[item.account_id]
item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
if item.reblog?
status_stat = status_stats[item.reblog.id]
item.reblog.status_stat = status_stat if status_stat.present?
else
status_stat = status_stats[item.id]
item.status_stat = status_stat if status_stat.present?
end
end
end

8
app/policies/admin/status_policy.rb
View File

@ -12,7 +12,7 @@ class Admin::StatusPolicy < ApplicationPolicy
end
def show?
role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported?)
role.can?(:manage_reports, :manage_users) && (record.public_visibility? || record.unlisted_visibility? || record.reported? || viewable_through_normal_policy?)
end
def destroy?
@ -26,4 +26,10 @@ class Admin::StatusPolicy < ApplicationPolicy
def review?
role.can?(:manage_taxonomies)
end
private
def viewable_through_normal_policy?
StatusPolicy.new(current_account, record, @preloaded_relations).show?
end
end

2
app/services/activitypub/fetch_remote_poll_service.rb
View File

@ -8,6 +8,6 @@ class ActivityPub::FetchRemotePollService < BaseService
return unless supported_context?(json)
ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json)
ActivityPub::ProcessStatusUpdateService.new.call(poll.status, json, json)
end
end

3
app/services/activitypub/process_account_service.rb
View File

@ -76,6 +76,9 @@ class ActivityPub::ProcessAccountService < BaseService
@account.suspended_at = domain_block.created_at if auto_suspend?
@account.suspension_origin = :local if auto_suspend?
@account.silenced_at = domain_block.created_at if auto_silence?
set_immediate_protocol_attributes!
@account.save
end

7
app/services/activitypub/process_status_update_service.rb
View File

@ -5,10 +5,11 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
include Redisable
include Lockable
def call(status, json, request_id: nil)
def call(status, activity_json, object_json, request_id: nil)
raise ArgumentError, 'Status has unsaved changes' if status.changed?
@json = json
@activity_json = activity_json
@json = object_json
@status_parser = ActivityPub::Parser::StatusParser.new(@json)
@uri = @status_parser.uri
@status = status
@ -308,6 +309,6 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
end
def forwarder
@forwarder ||= ActivityPub::Forwarder.new(@account, @json, @status)
@forwarder ||= ActivityPub::Forwarder.new(@account, @activity_json, @status)
end
end

4
app/services/translate_status_service.rb
View File

@ -12,7 +12,9 @@ class TranslateStatusService < BaseService
@content = status_content_format(@status)
@target_language = target_language
Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) }
Rails.cache.fetch("translations:v2/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do
translation_backend.translate(@content, @status.language, @target_language)
end
end
private

2
app/views/admin/trends/links/preview_card_providers/index.html.haml
View File

@ -29,7 +29,7 @@
- Trends::PreviewCardProviderFilter::KEYS.each do |key|
= hidden_field_tag key, params[key] if params[key].present?
.batch-table.optional
.batch-table
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false

2
config/initializers/content_security_policy.rb
View File

@ -3,7 +3,7 @@
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
def host_to_url(str)
"http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}".split('/').first if str.present?
"http#{Rails.configuration.x.use_https ? 's' : ''}://#{str.split('/').first}" if str.present?
end
base_host = Rails.configuration.x.web_domain

5
config/initializers/sidekiq.rb
View File

@ -3,6 +3,11 @@
require_relative '../../lib/mastodon/sidekiq_middleware'
Sidekiq.configure_server do |config|
if Rails.configuration.database_configuration.dig('production', 'adapter') == 'postgresql_makara'
STDERR.puts 'ERROR: Database replication is not currently supported in Sidekiq workers. Check your configuration.'
exit 1
end
config.redis = REDIS_SIDEKIQ_PARAMS
config.server_middleware do |chain|

2
config/routes.rb
View File

@ -292,7 +292,7 @@ Rails.application.routes.draw do
end
end
resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ } do
resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ }, format: 'html' do
member do
post :clear_delivery_errors
post :restart_delivery

6
docker-compose.yml
View File

@ -56,7 +56,7 @@ services:
web:
build: .
image: ghcr.io/mastodon/mastodon
image: ghcr.io/mastodon/mastodon:v4.1.9
restart: always
env_file: .env.production
command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@ -77,7 +77,7 @@ services:
streaming:
build: .
image: ghcr.io/mastodon/mastodon
image: ghcr.io/mastodon/mastodon:v4.1.9
restart: always
env_file: .env.production
command: node ./streaming
@ -95,7 +95,7 @@ services:
sidekiq:
build: .
image: ghcr.io/mastodon/mastodon
image: ghcr.io/mastodon/mastodon:v4.1.9
restart: always
env_file: .env.production
command: bundle exec sidekiq

2
lib/mastodon/version.rb
View File

@ -13,7 +13,7 @@ module Mastodon
end
def patch
4
9
end
def flags

12
lib/paperclip/transcoder.rb
View File

@ -37,12 +37,14 @@ module Paperclip
@output_options['f'] = 'image2'
@output_options['vframes'] = 1
when 'mp4'
@output_options['acodec'] = 'aac'
@output_options['strict'] = 'experimental'
unless eligible_to_passthrough?(metadata)
@output_options['acodec'] = 'aac'
@output_options['strict'] = 'experimental'
if high_vfr?(metadata) && !eligible_to_passthrough?(metadata)
@output_options['vsync'] = 'vfr'
@output_options['r'] = @vfr_threshold
if high_vfr?(metadata)
@output_options['vsync'] = 'vfr'
@output_options['r'] = @vfr_threshold
end
end
end

20
spec/controllers/admin/statuses_controller_spec.rb
View File

@ -40,24 +40,36 @@ describe Admin::StatusesController do
end
describe 'POST #batch' do
before do
post :batch, params: { account_id: account.id, action => '', admin_status_batch_action: { status_ids: status_ids } }
end
subject { post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } } }
let(:status_ids) { [media_attached_status.id] }
context 'when action is report' do
shared_examples 'when action is report' do
let(:action) { 'report' }
it 'creates a report' do
subject
report = Report.last
expect(report.target_account_id).to eq account.id
expect(report.status_ids).to eq status_ids
end
it 'redirects to report page' do
subject
expect(response).to redirect_to(admin_report_path(Report.last.id))
end
end
it_behaves_like 'when action is report'
context 'when the moderator is blocked by the author' do
before do
account.block!(user.account)
end
it_behaves_like 'when action is report'
end
end
end

66
spec/controllers/api/v1/timelines/tag_controller_spec.rb
View File

@ -5,36 +5,66 @@ require 'rails_helper'
describe Api::V1::Timelines::TagController do
render_views
let(:user) { Fabricate(:user) }
let(:user) { Fabricate(:user) }
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') }
before do
allow(controller).to receive(:doorkeeper_token) { token }
end
context 'with a user context' do
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id) }
describe 'GET #show' do
subject do
get :show, params: { id: 'test' }
end
describe 'GET #show' do
before do
PostStatusService.new.call(user.account, text: 'It is a #test')
before do
PostStatusService.new.call(user.account, text: 'It is a #test')
end
context 'when the instance allows public preview' do
context 'when the user is not authenticated' do
let(:token) { nil }
it 'returns http success', :aggregate_failures do
subject
expect(response).to have_http_status(200)
expect(response.headers['Link'].links.size).to eq(2)
end
end
it 'returns http success' do
get :show, params: { id: 'test' }
expect(response).to have_http_status(200)
expect(response.headers['Link'].links.size).to eq(2)
context 'when the user is authenticated' do
it 'returns http success', :aggregate_failures do
subject
expect(response).to have_http_status(200)
expect(response.headers['Link'].links.size).to eq(2)
end
end
end
end
context 'without a user context' do
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) }
context 'when the instance does not allow public preview' do
before do
Form::AdminSettings.new(timeline_preview: false).save
end
describe 'GET #show' do
it 'returns http success' do
get :show, params: { id: 'test' }
expect(response).to have_http_status(200)
expect(response.headers['Link']).to be_nil
context 'when the user is not authenticated' do
let(:token) { nil }
it 'returns http unauthorized' do
subject
expect(response).to have_http_status(401)
end
end
context 'when the user is authenticated' do
it 'returns http success', :aggregate_failures do
subject
expect(response).to have_http_status(200)
expect(response.headers['Link'].links.size).to eq(2)
end
end
end
end

24
spec/controllers/concerns/cache_concern_spec.rb
View File

@ -13,12 +13,17 @@ RSpec.describe CacheConcern, type: :controller do
def empty_relation
render plain: cache_collection(Status.none, Status).size
end
def account_statuses_favourites
render plain: cache_collection(Status.where(account_id: params[:id]), Status).map(&:favourites_count)
end
end
before do
routes.draw do
get 'empty_array' => 'anonymous#empty_array'
post 'empty_relation' => 'anonymous#empty_relation'
get 'empty_array' => 'anonymous#empty_array'
get 'empty_relation' => 'anonymous#empty_relation'
get 'account_statuses_favourites' => 'anonymous#account_statuses_favourites'
end
end
@ -36,5 +41,20 @@ RSpec.describe CacheConcern, type: :controller do
expect(response.body).to eq '0'
end
end
context 'when given a collection of statuses' do
let!(:account) { Fabricate(:account) }
let!(:status) { Fabricate(:status, account: account) }
it 'correctly updates with new interactions' do
get :account_statuses_favourites, params: { id: account.id }
expect(response.body).to eq '[0]'
FavouriteService.new.call(account, status)
get :account_statuses_favourites, params: { id: account.id }
expect(response.body).to eq '[1]'
end
end
end
end

31
spec/lib/activitypub/activity/flag_spec.rb
View File

@ -37,6 +37,37 @@ RSpec.describe ActivityPub::Activity::Flag do
end
end
context 'when the report comment is excessively long' do
subject do
described_class.new({
'@context': 'https://www.w3.org/ns/activitystreams',
id: flag_id,
type: 'Flag',
content: long_comment,
actor: ActivityPub::TagManager.instance.uri_for(sender),
object: [
ActivityPub::TagManager.instance.uri_for(flagged),
ActivityPub::TagManager.instance.uri_for(status),
],
}.with_indifferent_access, sender)
end
let(:long_comment) { Faker::Lorem.characters(number: 6000) }
before do
subject.perform
end
it 'creates a report but with a truncated comment' do
report = Report.find_by(account: sender, target_account: flagged)
expect(report).to_not be_nil
expect(report.comment.length).to eq 5000
expect(report.comment).to eq long_comment[0...5000]
expect(report.status_ids).to eq [status.id]
end
end
context 'when the reported status is private and should not be visible to the remote server' do
let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) }

11
spec/models/report_spec.rb
View File

@ -125,10 +125,17 @@ describe Report do
expect(report).to be_valid
end
it 'is invalid if comment is longer than 1000 characters' do
let(:remote_account) { Fabricate(:account, domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') }
it 'is invalid if comment is longer than 1000 characters only if reporter is local' do
report = Fabricate.build(:report, comment: Faker::Lorem.characters(number: 1001))
report.valid?
expect(report.valid?).to be false
expect(report).to model_have_error_on_field(:comment)
end
it 'is valid if comment is longer than 1000 characters and reporter is not local' do
report = Fabricate.build(:report, account: remote_account, comment: Faker::Lorem.characters(number: 1001))
expect(report.valid?).to be true
end
end
end

27
spec/requests/content_security_policy_spec.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
require 'rails_helper'
describe 'Content-Security-Policy' do
it 'sets the expected CSP headers' do
allow(SecureRandom).to receive(:base64).with(16).and_return('ZbA+JmE7+bK8F5qvADZHuQ==')
get '/'
expect(response.headers['Content-Security-Policy'].split(';').map(&:strip)).to contain_exactly(
"base-uri 'none'",
"default-src 'none'",
"frame-ancestors 'none'",
"font-src 'self' https://cb6e6126.ngrok.io",
"img-src 'self' https: data: blob: https://cb6e6126.ngrok.io",
"style-src 'self' https://cb6e6126.ngrok.io 'nonce-ZbA+JmE7+bK8F5qvADZHuQ=='",
"media-src 'self' https: data: https://cb6e6126.ngrok.io",
"frame-src 'self' https:",
"manifest-src 'self' https://cb6e6126.ngrok.io",
"form-action 'self'",
"child-src 'self' blob: https://cb6e6126.ngrok.io",
"worker-src 'self' blob: https://cb6e6126.ngrok.io",
"connect-src 'self' data: blob: https://cb6e6126.ngrok.io https://cb6e6126.ngrok.io ws://localhost:4000",
"script-src 'self' https://cb6e6126.ngrok.io 'wasm-unsafe-eval'"
)
end
end

40
spec/services/activitypub/process_status_update_service_spec.rb
View File

@ -41,12 +41,12 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
describe '#call' do
it 'updates text' do
subject.call(status, json)
subject.call(status, json, json)
expect(status.reload.text).to eq 'Hello universe'
end
it 'updates content warning' do
subject.call(status, json)
subject.call(status, json, json)
expect(status.reload.spoiler_text).to eq 'Show more'
end
@ -64,7 +64,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'does not create any edits' do
@ -87,7 +87,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'does not create any edits' do
@ -135,7 +135,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'does not create any edits' do
@ -188,7 +188,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'does not create any edits' do
@ -216,11 +216,11 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
it 'does not create any edits' do
expect { subject.call(status, json) }.not_to change { status.reload.edits.pluck(&:id) }
expect { subject.call(status, json, json) }.to_not(change { status.reload.edits.pluck(&:id) })
end
it 'does not update the text, spoiler_text or edited_at' do
expect { subject.call(status, json) }.not_to change { s = status.reload; [s.text, s.spoiler_text, s.edited_at] }
expect { subject.call(status, json, json) }.to_not(change { s = status.reload; [s.text, s.spoiler_text, s.edited_at] })
end
end
@ -235,7 +235,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'does not create any edits' do
@ -259,7 +259,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
before do
status.update(ordered_media_attachment_ids: nil)
subject.call(status, json)
subject.call(status, json, json)
end
it 'does not create any edits' do
@ -273,7 +273,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
context 'originally without tags' do
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'updates tags' do
@ -299,7 +299,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'updates tags' do
@ -309,7 +309,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
context 'originally without mentions' do
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'updates mentions' do
@ -321,7 +321,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
let(:mentions) { [alice, bob] }
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'updates mentions' do
@ -332,7 +332,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
context 'originally without media attachments' do
before do
stub_request(:get, 'https://example.com/foo.png').to_return(body: attachment_fixture('emojo.png'))
subject.call(status, json)
subject.call(status, json, json)
end
let(:payload) do
@ -382,7 +382,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
before do
allow(RedownloadMediaWorker).to receive(:perform_async)
subject.call(status, json)
subject.call(status, json, json)
end
it 'updates the existing media attachment in-place' do
@ -410,7 +410,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
before do
poll = Fabricate(:poll, status: status)
status.update(preloadable_poll: poll)
subject.call(status, json)
subject.call(status, json, json)
end
it 'removes poll' do
@ -440,7 +440,7 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
before do
subject.call(status, json)
subject.call(status, json, json)
end
it 'creates a poll' do
@ -456,12 +456,12 @@ RSpec.describe ActivityPub::ProcessStatusUpdateService, type: :service do
end
it 'creates edit history' do
subject.call(status, json)
subject.call(status, json, json)
expect(status.edits.reload.map(&:text)).to eq ['Hello world', 'Hello universe']
end
it 'sets edited timestamp' do
subject.call(status, json)
subject.call(status, json, json)
expect(status.reload.edited_at.to_s).to eq '2021-09-08 22:39:25 UTC'
end
end

8
spec/services/report_service_spec.rb
View File

@ -4,6 +4,14 @@ RSpec.describe ReportService, type: :service do
subject { described_class.new }
let(:source_account) { Fabricate(:account) }
let(:target_account) { Fabricate(:account) }
context 'with a local account' do
it 'has a uri' do
report = subject.call(source_account, target_account)
expect(report.uri).to_not be_nil
end
end
context 'for a remote account' do
let(:remote_account) { Fabricate(:account, domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') }

202
streaming/index.js
View File

@ -226,9 +226,15 @@ const startWorker = async (workerId) => {
callbacks.forEach(callback => callback(json));
};
/**
* @callback SubscriptionListener
* @param {ReturnType<parseJSON>} json of the message
* @returns void
*/
/**
* @param {string} channel
* @param {function(string): void} callback
* @param {SubscriptionListener} callback
*/
const subscribe = (channel, callback) => {
log.silly(`Adding listener for ${channel}`);
@ -245,7 +251,7 @@ const startWorker = async (workerId) => {
/**
* @param {string} channel
* @param {function(Object<string, any>): void} callback
* @param {SubscriptionListener} callback
*/
const unsubscribe = (channel, callback) => {
log.silly(`Removing listener for ${channel}`);
@ -623,51 +629,66 @@ const startWorker = async (workerId) => {
* @param {string[]} ids
* @param {any} req
* @param {function(string, string): void} output
* @param {function(string[], function(string): void): void} attachCloseHandler
* @param {undefined | function(string[], SubscriptionListener): void} attachCloseHandler
* @param {boolean=} needsFiltering
* @returns {function(object): void}
* @returns {SubscriptionListener}
*/
const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false) => {
const accountId = req.accountId || req.remoteAddress;
log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`);
// Currently message is of type string, soon it'll be Record<string, any>
const transmit = (event, payload) => {
// TODO: Replace "string"-based delete payloads with object payloads:
const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload;
log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload}`);
output(event, encodedPayload);
};
// The listener used to process each message off the redis subscription,
// message here is an object with an `event` and `payload` property. Some
// events also include a queued_at value, but this is being removed shortly.
/** @type {SubscriptionListener} */
const listener = message => {
const { event, payload, queued_at } = message;
const { event, payload } = message;
const transmit = () => {
const now = new Date().getTime();
const delta = now - queued_at;
const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload;
log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload} Delay: ${delta}ms`);
output(event, encodedPayload);
};
// Only messages that may require filtering are statuses, since notifications
// are already personalized and deletes do not matter
if (!needsFiltering || event !== 'update') {
transmit();
// Streaming only needs to apply filtering to some channels and only to
// some events. This is because majority of the filtering happens on the
// Ruby on Rails side when producing the event for streaming.
//
// The only events that require filtering from the streaming server are
// `update` and `status.update`, all other events are transmitted to the
// client as soon as they're received (pass-through).
//
// The channels that need filtering are determined in the function
// `channelNameToIds` defined below:
if (!needsFiltering || (event !== 'update' && event !== 'status.update')) {
transmit(event, payload);
return;
}
const unpackedPayload = payload;
const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id));
const accountDomain = unpackedPayload.account.acct.split('@')[1];
// The rest of the logic from here on in this function is to handle
// filtering of statuses:
if (Array.isArray(req.chosenLanguages) && unpackedPayload.language !== null && req.chosenLanguages.indexOf(unpackedPayload.language) === -1) {
log.silly(req.requestId, `Message ${unpackedPayload.id} filtered by language (${unpackedPayload.language})`);
// Filter based on language:
if (Array.isArray(req.chosenLanguages) && payload.language !== null && req.chosenLanguages.indexOf(payload.language) === -1) {
log.silly(req.requestId, `Message ${payload.id} filtered by language (${payload.language})`);
return;
}
// When the account is not logged in, it is not necessary to confirm the block or mute
if (!req.accountId) {
transmit();
transmit(event, payload);
return;
}
pgPool.connect((err, client, done) => {
// Filter based on domain blocks, blocks, mutes, or custom filters:
const targetAccountIds = [payload.account.id].concat(payload.mentions.map(item => item.id));
const accountDomain = payload.account.acct.split('@')[1];
// TODO: Move this logic out of the message handling loop
pgPool.connect((err, client, releasePgConnection) => {
if (err) {
log.error(err);
return;
@ -682,40 +703,57 @@ const startWorker = async (workerId) => {
SELECT 1
FROM mutes
WHERE account_id = $1
AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)),
AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, payload.account.id].concat(targetAccountIds)),
];
if (accountDomain) {
queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain]));
}
if (!unpackedPayload.filtered && !req.cachedFilters) {
if (!payload.filtered && !req.cachedFilters) {
queries.push(client.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())', [req.accountId]));
}
Promise.all(queries).then(values => {
done();
releasePgConnection();
// Handling blocks & mutes and domain blocks: If one of those applies,
// then we don't transmit the payload of the event to the client
if (values[0].rows.length > 0 || (accountDomain && values[1].rows.length > 0)) {
return;
}
if (!unpackedPayload.filtered && !req.cachedFilters) {
// If the payload already contains the `filtered` property, it means
// that filtering has been applied on the ruby on rails side, as
// such, we don't need to construct or apply the filters in streaming:
if (Object.prototype.hasOwnProperty.call(payload, "filtered")) {
transmit(event, payload);
return;
}
// Handling for constructing the custom filters and caching them on the request
// TODO: Move this logic out of the message handling lifecycle
if (!req.cachedFilters) {
const filterRows = values[accountDomain ? 2 : 1].rows;
req.cachedFilters = filterRows.reduce((cache, row) => {
if (cache[row.id]) {
cache[row.id].keywords.push([row.keyword, row.whole_word]);
req.cachedFilters = filterRows.reduce((cache, filter) => {
if (cache[filter.id]) {
cache[filter.id].keywords.push([filter.keyword, filter.whole_word]);
} else {
cache[row.id] = {
keywords: [[row.keyword, row.whole_word]],
expires_at: row.expires_at,
repr: {
id: row.id,
title: row.title,
context: row.context,
expires_at: row.expires_at,
filter_action: ['warn', 'hide'][row.filter_action],
cache[filter.id] = {
keywords: [[filter.keyword, filter.whole_word]],
expires_at: filter.expires_at,
filter: {
id: filter.id,
title: filter.title,
context: filter.context,
expires_at: filter.expires_at,
// filter.filter_action is the value from the
// custom_filters.action database column, it is an integer
// representing a value in an enum defined by Ruby on Rails:
//
// enum { warn: 0, hide: 1 }
filter_action: ['warn', 'hide'][filter.filter_action],
},
};
}
@ -723,6 +761,10 @@ const startWorker = async (workerId) => {
return cache;
}, {});
// Construct the regular expressions for the custom filters: This
// needs to be done in a separate loop as the database returns one
// filterRow per keyword, so we need all the keywords before
// constructing the regular expression
Object.keys(req.cachedFilters).forEach((key) => {
req.cachedFilters[key].regexp = new RegExp(req.cachedFilters[key].keywords.map(([keyword, whole_word]) => {
let expr = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
@ -742,31 +784,58 @@ const startWorker = async (workerId) => {
});
}
// Check filters
if (req.cachedFilters && !unpackedPayload.filtered) {
const status = unpackedPayload;
const searchContent = ([status.spoiler_text || '', status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
const searchIndex = JSDOM.fragment(searchContent).textContent;
// Apply cachedFilters against the payload, constructing a
// `filter_results` array of FilterResult entities
if (req.cachedFilters) {
const status = payload;
// TODO: Calculate searchableContent in Ruby on Rails:
const searchableContent = ([status.spoiler_text || '', status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
const searchableTextContent = JSDOM.fragment(searchableContent).textContent;
const now = new Date();
payload.filtered = [];
Object.values(req.cachedFilters).forEach((cachedFilter) => {
if ((cachedFilter.expires_at === null || cachedFilter.expires_at > now)) {
const keyword_matches = searchIndex.match(cachedFilter.regexp);
if (keyword_matches) {
payload.filtered.push({
filter: cachedFilter.repr,
keyword_matches,
});
}
const filter_results = Object.values(req.cachedFilters).reduce((results, cachedFilter) => {
// Check the filter hasn't expired before applying:
if (cachedFilter.expires_at !== null && cachedFilter.expires_at < now) {
return results;
}
});
}
transmit();
// Just in-case JSDOM fails to find textContent in searchableContent
if (!searchableTextContent) {
return results;
}
const keyword_matches = searchableTextContent.match(cachedFilter.regexp);
if (keyword_matches) {
// results is an Array of FilterResult; status_matches is always
// null as we only are only applying the keyword-based custom
// filters, not the status-based custom filters.
// https://docs.joinmastodon.org/entities/FilterResult/
results.push({
filter: cachedFilter.filter,
keyword_matches,
status_matches: null
});
}
return results;
}, []);
// Send the payload + the FilterResults as the `filtered` property
// to the streaming connection. To reach this code, the `event` must
// have been either `update` or `status.update`, meaning the
// `payload` is a Status entity, which has a `filtered` property:
//
// filtered: https://docs.joinmastodon.org/entities/Status/#filtered
transmit(event, {
...payload,
filtered: filter_results
});
} else {
transmit(event, payload);
}
}).catch(err => {
releasePgConnection();
log.error(err);
done();
});
});
};
@ -775,7 +844,7 @@ const startWorker = async (workerId) => {
subscribe(`${redisPrefix}${id}`, listener);
});
if (attachCloseHandler) {
if (typeof attachCloseHandler === 'function') {
attachCloseHandler(ids.map(id => `${redisPrefix}${id}`), listener);
}
@ -812,12 +881,13 @@ const startWorker = async (workerId) => {
/**
* @param {any} req
* @param {function(): void} [closeHandler]
* @return {function(string[]): void}
* @returns {function(string[], SubscriptionListener): void}
*/
const streamHttpEnd = (req, closeHandler = undefined) => (ids) => {
const streamHttpEnd = (req, closeHandler = undefined) => (ids, listener) => {
req.on('close', () => {
ids.forEach(id => {
unsubscribe(id);
unsubscribe(id, listener);
});
if (closeHandler) {
@ -1077,7 +1147,7 @@ const startWorker = async (workerId) => {
* @typedef WebSocketSession
* @property {any} socket
* @property {any} request
* @property {Object.<string, { listener: function(string): void, stopHeartbeat: function(): void }>} subscriptions
* @property {Object.<string, { listener: SubscriptionListener, stopHeartbeat: function(): void }>} subscriptions
*/
/**