Merge remote-tracking branch 'upstream/stable-4.1' into 4.1.10

CHANGELOG.md

@@ -3,6 +3,28 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.18] - 2024-07-04
+
+### Security
+
+- Fix incorrect permission checking on multiple API endpoints ([GHSA-58x8-3qxw-6hm7](https://github.com/mastodon/mastodon/security/advisories/GHSA-58x8-3qxw-6hm7))
+- Fix incorrect authorship checking when processing some activities (CVE-2024-37903, [GHSA-xjvf-fm67-4qc3](https://github.com/mastodon/mastodon/security/advisories/GHSA-xjvf-fm67-4qc3))
+- Fix ongoing streaming sessions not being invalidated when application tokens get revoked ([GHSA-vp5r-5pgw-jwqx](https://github.com/mastodon/mastodon/security/advisories/GHSA-vp5r-5pgw-jwqx))
+- Update dependencies
+
+### Changed
+
+- Change preview cards generation to skip unusually long URLs ([oneiros](https://github.com/mastodon/mastodon/pull/30854))
+- Change search modifiers to be case-insensitive ([Gargron](https://github.com/mastodon/mastodon/pull/30865))
+- Change `STATSD_ADDR` handling to emit a warning rather than crashing if the address is unreachable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30691))
+- Change PWA start URL from `/home` to `/` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27377))
+
+### Fixed
+
+- Fix scheduled statuses scheduled in less than 5 minutes being immediately published ([danielmbrasil](https://github.com/mastodon/mastodon/pull/30584))
+- Fix encoding detection for link cards ([oneiros](https://github.com/mastodon/mastodon/pull/30780))
+- Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30819))
+
 ## [4.1.17] - 2024-05-30
 
 ### Security

Gemfile.lock

@@ -10,40 +10,40 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actioncable (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionmailbox (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       mail (>= 2.7.1)
-    actionmailer (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      actionview (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionmailer (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      actionview (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7.7)
-      actionview (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionpack (6.1.7.8)
+      actionview (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actiontext (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7.7)
-      activesupport (= 6.1.7.7)
+    actionview (6.1.7.8)
+      activesupport (= 6.1.7.8)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -54,22 +54,22 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.8)
-    activejob (6.1.7.7)
-      activesupport (= 6.1.7.7)
+    activejob (6.1.7.8)
+      activesupport (= 6.1.7.8)
       globalid (>= 0.3.6)
-    activemodel (6.1.7.7)
-      activesupport (= 6.1.7.7)
-    activerecord (6.1.7.7)
-      activemodel (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
-    activestorage (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    activemodel (6.1.7.8)
+      activesupport (= 6.1.7.8)
+    activerecord (6.1.7.8)
+      activemodel (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
+    activestorage (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7.7)
+    activesupport (6.1.7.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -105,6 +105,7 @@ GEM
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.5.2)
       aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.2.0)
     bcrypt (3.1.17)
     better_errors (2.9.1)
       coderay (>= 1.0.0)
@@ -396,7 +397,7 @@ GEM
       net-smtp
     makara (0.5.1)
       activerecord (>= 5.2.0)
-    marcel (1.0.2)
+    marcel (1.0.4)
     mario-redis-lock (1.2.1)
       redis (>= 3.0.5)
     matrix (0.4.2)
@@ -406,7 +407,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.7)
     minitest (5.17.0)
     msgpack (1.6.0)
     multi_json (1.15.0)
@@ -425,7 +426,7 @@ GEM
       net-protocol
     net-ssh (7.0.1)
     nio4r (2.5.9)
-    nokogiri (1.16.5)
+    nokogiri (1.16.6)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.2.8)
@@ -498,7 +499,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.7.3)
-    rack (2.2.8.1)
+    rack (2.2.9)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
@@ -513,20 +514,20 @@ GEM
       rack
     rack-test (2.0.2)
       rack (>= 1.3)
-    rails (6.1.7.7)
-      actioncable (= 6.1.7.7)
-      actionmailbox (= 6.1.7.7)
-      actionmailer (= 6.1.7.7)
-      actionpack (= 6.1.7.7)
-      actiontext (= 6.1.7.7)
-      actionview (= 6.1.7.7)
-      activejob (= 6.1.7.7)
-      activemodel (= 6.1.7.7)
-      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    rails (6.1.7.8)
+      actioncable (= 6.1.7.8)
+      actionmailbox (= 6.1.7.8)
+      actionmailer (= 6.1.7.8)
+      actionpack (= 6.1.7.8)
+      actiontext (= 6.1.7.8)
+      actionview (= 6.1.7.8)
+      activejob (= 6.1.7.8)
+      activemodel (= 6.1.7.8)
+      activerecord (= 6.1.7.8)
+      activestorage (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       bundler (>= 1.15.0)
-      railties (= 6.1.7.7)
+      railties (= 6.1.7.8)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -542,9 +543,9 @@ GEM
       railties (>= 6.0.0, < 7)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (6.1.7.7)
-      actionpack (= 6.1.7.7)
-      activesupport (= 6.1.7.7)
+    railties (6.1.7.8)
+      actionpack (= 6.1.7.8)
+      activesupport (= 6.1.7.8)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -665,7 +666,8 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     smart_properties (1.17.0)
-    sprockets (3.7.2)
+    sprockets (3.7.3)
+      base64
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.4.2)
@@ -756,7 +758,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   ruby

app/controllers/api/v1/scheduled_statuses_controller.rb

@@ -6,6 +6,7 @@ class Api::V1::ScheduledStatusesController < Api::BaseController
   before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, except: [:update, :destroy]
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:update, :destroy]
 
+  before_action :require_user!
   before_action :set_statuses, only: :index
   before_action :set_status, except: :index
 

app/controllers/api/v1/statuses/translations_controller.rb

@@ -4,6 +4,7 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController
   include Authorization
 
   before_action -> { doorkeeper_authorize! :read, :'read:statuses' }
+  before_action :require_user!
   before_action :set_status
   before_action :set_translation
 

app/controllers/api/v1/timelines/public_controller.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::PublicController < Api::BaseController
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
+  before_action :require_user!, only: [:show], if: :require_auth?
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
   def show

app/controllers/api/v1/timelines/tag_controller.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::TagController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth?
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
+  before_action :require_user!, if: :require_auth?
   before_action :load_tag
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 

app/controllers/oauth/authorized_applications_controller.rb

@@ -16,6 +16,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
 
   def destroy
     Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner)
+    Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner)
     super
   end
 

app/lib/activitypub/activity/create.rb

@@ -110,7 +110,7 @@ def distribute
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
-    status
+    status if status&.account_id == @account.id
   end
 
   def process_status_params

app/lib/application_extension.rb

@@ -14,17 +14,19 @@ module ApplicationExtension
     # dependent: delete_all, which means the ActiveRecord callback in
     # AccessTokenExtension is not run, so instead we manually announce to
     # streaming that these tokens are being deleted.
-    before_destroy :push_to_streaming_api, prepend: true
+    before_destroy :close_streaming_sessions, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
 
-  def push_to_streaming_api
+  def close_streaming_sessions(resource_owner = nil)
     # TODO: #28793 Combine into a single topic
     payload = Oj.dump(event: :kill)
-    access_tokens.in_batches do |tokens|
+    scope = access_tokens
+    scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil?
+    scope.in_batches do |tokens|
       redis.pipelined do |pipeline|
         tokens.ids.each do |id|
           pipeline.publish("timeline:access_token:#{id}", payload)

app/lib/link_details_extractor.rb

@@ -255,16 +255,21 @@ def structured_data
   end
 
   def document
-    @document ||= Nokogiri::HTML(@html, nil, encoding)
+    @document ||= detect_encoding_and_parse_document
   end
 
-  def encoding
-    @encoding ||= begin
-      guess = detector.detect(@html, @html_charset)
-      guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  def detect_encoding_and_parse_document
+    [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
+      document = Nokogiri::HTML(@html, nil, encoding)
+      return document if document.to_s.valid_encoding?
     end
   end
 
+  def detect_encoding
+    guess = detector.detect(@html, @html_charset)
+    guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  end
+
   def detector
     @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
       detector.strip_tags = true

app/lib/search_query_transformer.rb

@@ -101,7 +101,7 @@ def initialize(prefix, term)
   end
 
   rule(clause: subtree(:clause)) do
-    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    prefix   = clause[:prefix][:term].to_s.downcase if clause[:prefix]
     operator = clause[:operator]&.to_s
 
     if clause[:prefix]

app/models/status_edit.rb

@@ -41,7 +41,7 @@ class PreservedMediaAttachment < ActiveModelSerializers::Model
   default_scope { order(id: :asc) }
 
   delegate :local?, :application, :edited?, :edited_at,
-           :discarded?, :visibility, to: :status
+           :discarded?, :visibility, :language, to: :status
 
   def emojis
     return @emojis if defined?(@emojis)

app/serializers/manifest_serializer.rb

@@ -16,11 +16,18 @@ class ManifestSerializer < ActiveModel::Serializer
     512
   ).freeze
 
-  attributes :name, :short_name,
+  attributes :id, :name, :short_name,
              :icons, :theme_color, :background_color,
              :display, :start_url, :scope,
              :share_target, :shortcuts
 
+  def id
+    # This is set to `/home` because that was the old value of `start_url` and
+    # thus the fallback ID computed by Chrome:
+    # https://developer.chrome.com/blog/pwa-manifest-id/
+    '/home'
+  end
+
   def name
     object.title
   end
@@ -53,7 +60,7 @@ def display
   end
 
   def start_url
-    '/home'
+    '/'
   end
 
   def scope