From 40b69cc1cd62fa57047905a6999985ae6c183f6c Mon Sep 17 00:00:00 2001 From: Stanislas Signoud Date: Thu, 31 Aug 2023 12:18:46 +0200 Subject: [PATCH 01/16] Add an explanation banner on switching to single column mode (#26019) --- .../features/ui/components/navigation_panel.jsx | 16 +++++++++++----- app/javascript/mastodon/locales/en.json | 1 + app/javascript/mastodon/locales/fr.json | 1 + app/javascript/styles/mastodon/components.scss | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.jsx b/app/javascript/mastodon/features/ui/components/navigation_panel.jsx index d36abf8f1..8006ca89a 100644 --- a/app/javascript/mastodon/features/ui/components/navigation_panel.jsx +++ b/app/javascript/mastodon/features/ui/components/navigation_panel.jsx @@ -31,6 +31,7 @@ const messages = defineMessages({ about: { id: 'navigation_bar.about', defaultMessage: 'About' }, search: { id: 'navigation_bar.search', defaultMessage: 'Search' }, advancedInterface: { id: 'navigation_bar.advanced_interface', defaultMessage: 'Open in advanced web interface' }, + openedInClassicInterface: { id: 'navigation_bar.opened_in_classic_interface', defaultMessage: 'Posts, accounts, and other specific pages are opened by default in the classic web interface.' }, }); class NavigationPanel extends Component { @@ -57,12 +58,17 @@ class NavigationPanel extends Component {
- {transientSingleColumn && ( - - {intl.formatMessage(messages.advancedInterface)} - + {transientSingleColumn ? ( +
+ {intl.formatMessage(messages.openedInClassicInterface)} + {" "} + + {intl.formatMessage(messages.advancedInterface)} + +
+ ) : ( +
)} -
{signedIn && ( diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json index 5871b08de..90bb9616f 100644 --- a/app/javascript/mastodon/locales/en.json +++ b/app/javascript/mastodon/locales/en.json @@ -411,6 +411,7 @@ "navigation_bar.lists": "Lists", "navigation_bar.logout": "Logout", "navigation_bar.mutes": "Muted users", + "navigation_bar.opened_in_classic_interface": "Posts, accounts, and other specific pages are opened by default in the classic web interface.", "navigation_bar.personal": "Personal", "navigation_bar.pins": "Pinned posts", "navigation_bar.preferences": "Preferences", diff --git a/app/javascript/mastodon/locales/fr.json b/app/javascript/mastodon/locales/fr.json index 2bef3bb4b..116ed66d0 100644 --- a/app/javascript/mastodon/locales/fr.json +++ b/app/javascript/mastodon/locales/fr.json @@ -409,6 +409,7 @@ "navigation_bar.lists": "Listes", "navigation_bar.logout": "Déconnexion", "navigation_bar.mutes": "Comptes masqués", + "navigation_bar.opened_in_classic_interface": "Les messages, les comptes et les pages spécifiques sont ouvertes dans l’interface classique.", "navigation_bar.personal": "Personnel", "navigation_bar.pins": "Messages épinglés", "navigation_bar.preferences": "Préférences", diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index c07f95f56..95e6667b5 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -3270,6 +3270,22 @@ $ui-header-height: 55px; border-color: $ui-highlight-color; } +.switch-to-advanced { + color: $classic-primary-color; + background-color: $classic-base-color; + padding: 15px; + border-radius: 4px; + margin-top: 4px; + margin-bottom: 12px; + font-size: 13px; + line-height: 18px; + + .switch-to-advanced__toggle { + color: $ui-button-tertiary-color; + font-weight: bold; + } +} + .column-link { background: lighten($ui-base-color, 8%); color: $primary-text-color; From 336ec503c28356a969e2987ef426063f0943b4e3 Mon Sep 17 00:00:00 2001 From: Tyler Deitz Date: Thu, 31 Aug 2023 04:46:27 -0700 Subject: [PATCH 02/16] Add avatar image to webfinger responses (#26558) --- app/serializers/webfinger_serializer.rb | 37 +++++++---- .../well_known/webfinger_controller_spec.rb | 64 +++++++++++++++++++ 2 files changed, 89 insertions(+), 12 deletions(-) diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb index 3ca344116..b67cd2771 100644 --- a/app/serializers/webfinger_serializer.rb +++ b/app/serializers/webfinger_serializer.rb @@ -18,18 +18,31 @@ class WebfingerSerializer < ActiveModel::Serializer end def links - if object.instance_actor? - [ - { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) }, - { rel: 'self', type: 'application/activity+json', href: instance_actor_url }, - { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" }, - ] - else - [ - { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) }, - { rel: 'self', type: 'application/activity+json', href: account_url(object) }, - { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" }, - ] + [ + { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: profile_page_href }, + { rel: 'self', type: 'application/activity+json', href: self_href }, + { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" }, + ].tap do |x| + x << { rel: 'http://webfinger.net/rel/avatar', type: object.avatar.content_type, href: full_asset_url(object.avatar_original_url) } if show_avatar? end end + + private + + def show_avatar? + media_present = object.avatar.present? && object.avatar.content_type.present? + + # Show avatar only if an instance shows profiles to logged out users + allowed_by_config = ENV['DISALLOW_UNAUTHENTICATED_API_ACCESS'] != 'true' && !Rails.configuration.x.limited_federation_mode + + media_present && allowed_by_config + end + + def profile_page_href + object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object) + end + + def self_href + object.instance_actor? ? instance_actor_url : account_url(object) + end end diff --git a/spec/controllers/well_known/webfinger_controller_spec.rb b/spec/controllers/well_known/webfinger_controller_spec.rb index 8dc0f329b..20770a721 100644 --- a/spec/controllers/well_known/webfinger_controller_spec.rb +++ b/spec/controllers/well_known/webfinger_controller_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' describe WellKnown::WebfingerController do + include RoutingHelper + render_views describe 'GET #show' do @@ -167,5 +169,67 @@ describe WellKnown::WebfingerController do expect(response).to have_http_status(400) end end + + context 'when an account has an avatar' do + let(:alice) { Fabricate(:account, username: 'alice', avatar: attachment_fixture('attachment.jpg')) } + let(:resource) { alice.to_webfinger_s } + + it 'returns avatar in response' do + perform_show! + + avatar_link = get_avatar_link(body_as_json) + expect(avatar_link).to_not be_nil + expect(avatar_link[:type]).to eq alice.avatar.content_type + expect(avatar_link[:href]).to eq full_asset_url(alice.avatar) + end + + context 'with limited federation mode' do + before do + allow(Rails.configuration.x).to receive(:limited_federation_mode).and_return(true) + end + + it 'does not return avatar in response' do + perform_show! + + avatar_link = get_avatar_link(body_as_json) + expect(avatar_link).to be_nil + end + end + + context 'when enabling DISALLOW_UNAUTHENTICATED_API_ACCESS' do + around do |example| + ClimateControl.modify DISALLOW_UNAUTHENTICATED_API_ACCESS: 'true' do + example.run + end + end + + it 'does not return avatar in response' do + perform_show! + + avatar_link = get_avatar_link(body_as_json) + expect(avatar_link).to be_nil + end + end + end + + context 'when an account does not have an avatar' do + let(:alice) { Fabricate(:account, username: 'alice', avatar: nil) } + let(:resource) { alice.to_webfinger_s } + + before do + perform_show! + end + + it 'does not return avatar in response' do + avatar_link = get_avatar_link(body_as_json) + expect(avatar_link).to be_nil + end + end + end + + private + + def get_avatar_link(json) + json[:links].find { |link| link[:rel] == 'http://webfinger.net/rel/avatar' } end end From ccca542db1e73610a86c1d34d8b42f3f4a818af9 Mon Sep 17 00:00:00 2001 From: Daniel M Brasil Date: Thu, 31 Aug 2023 08:53:24 -0300 Subject: [PATCH 03/16] Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled (#26237) --- .../api/v1/timelines/tag_controller.rb | 5 ++ .../api/v1/timelines/tag_controller_spec.rb | 66 ++++++++++++++----- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb index 9cd7b9904..a79d65c12 100644 --- a/app/controllers/api/v1/timelines/tag_controller.rb +++ b/app/controllers/api/v1/timelines/tag_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Api::V1::Timelines::TagController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth? before_action :load_tag after_action :insert_pagination_headers, unless: -> { @statuses.empty? } @@ -12,6 +13,10 @@ class Api::V1::Timelines::TagController < Api::BaseController private + def require_auth? + !Setting.timeline_preview + end + def load_tag @tag = Tag.find_normalized(params[:id]) end diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb index 718911083..1c60798fc 100644 --- a/spec/controllers/api/v1/timelines/tag_controller_spec.rb +++ b/spec/controllers/api/v1/timelines/tag_controller_spec.rb @@ -5,36 +5,66 @@ require 'rails_helper' describe Api::V1::Timelines::TagController do render_views - let(:user) { Fabricate(:user) } + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } before do allow(controller).to receive(:doorkeeper_token) { token } end - context 'with a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id) } + describe 'GET #show' do + subject do + get :show, params: { id: 'test' } + end - describe 'GET #show' do - before do - PostStatusService.new.call(user.account, text: 'It is a #test') + before do + PostStatusService.new.call(user.account, text: 'It is a #test') + end + + context 'when the instance allows public preview' do + context 'when the user is not authenticated' do + let(:token) { nil } + + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end - it 'returns http success' do - get :show, params: { id: 'test' } - expect(response).to have_http_status(200) - expect(response.headers['Link'].links.size).to eq(2) + context 'when the user is authenticated' do + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end end - end - context 'without a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) } + context 'when the instance does not allow public preview' do + before do + Form::AdminSettings.new(timeline_preview: false).save + end + + context 'when the user is not authenticated' do + let(:token) { nil } + + it 'returns http unauthorized' do + subject + + expect(response).to have_http_status(401) + end + end + + context 'when the user is authenticated' do + it 'returns http success', :aggregate_failures do + subject - describe 'GET #show' do - it 'returns http success' do - get :show, params: { id: 'test' } - expect(response).to have_http_status(200) - expect(response.headers['Link']).to be_nil + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end end end From f1d250135ccf9be5c4d982a2c48417da89d38eb5 Mon Sep 17 00:00:00 2001 From: Santiago Kozak Date: Thu, 31 Aug 2023 09:15:58 -0300 Subject: [PATCH 04/16] Allow filter form in profiles directory to wrap (#26682) --- app/javascript/styles/mastodon/components.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 95e6667b5..f61cd059f 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -2381,6 +2381,7 @@ $ui-header-height: 55px; .filter-form { display: flex; + flex-wrap: wrap; } .autosuggest-textarea__textarea { From cffc5d2b0180e2ead23f32391246cdac357bf71b Mon Sep 17 00:00:00 2001 From: Renaud Chaput Date: Thu, 31 Aug 2023 14:59:50 +0200 Subject: [PATCH 05/16] Fix comment in build image workflow (#26740) --- .github/workflows/build-container-image.yml | 2 -- .github/workflows/build-releases.yml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml index 897bb9caa..a1aeddf20 100644 --- a/.github/workflows/build-container-image.yml +++ b/.github/workflows/build-container-image.yml @@ -76,8 +76,6 @@ jobs: if: ${{ inputs.push_to_images != '' }} with: images: ${{ inputs.push_to_images }} - # Only tag with latest when ran against the latest stable branch - # This needs to be updated after each minor version release flavor: ${{ inputs.flavor }} tags: ${{ inputs.tags }} labels: ${{ inputs.labels }} diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml index b40817468..f739a69d9 100644 --- a/.github/workflows/build-releases.yml +++ b/.github/workflows/build-releases.yml @@ -17,6 +17,8 @@ jobs: push_to_images: | tootsuite/mastodon ghcr.io/mastodon/mastodon + # Only tag with latest when ran against the latest stable branch + # This needs to be updated after each minor version release flavor: | latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') }} tags: | From 6b58cfd8dd2f635dbb1b9d10b6b243d20ba23535 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 31 Aug 2023 15:35:58 +0200 Subject: [PATCH 06/16] Fix searching by username by reverting account verbatim tokenizer to `standard` (#26739) --- app/chewy/accounts_index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index 8881b08f6..00db257ac 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -34,7 +34,7 @@ class AccountsIndex < Chewy::Index }, verbatim: { - tokenizer: 'uax_url_email', + tokenizer: 'standard', filter: %w(lowercase asciifolding cjk_width), }, From 1471be82255d948460b0b653dd48cdfbc67d90cf Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 31 Aug 2023 15:47:30 +0200 Subject: [PATCH 07/16] Fix `AddUniqueIndexOnPreviewCardsStatuses` migration requiring PostgreSQL 12+ in some cases (#26737) --- ...1_add_unique_index_on_preview_cards_statuses.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb b/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb index c35ad8002..3e9ab134b 100644 --- a/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb +++ b/db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb @@ -15,10 +15,22 @@ class AddUniqueIndexOnPreviewCardsStatuses < ActiveRecord::Migration[6.1] private + def supports_concurrent_reindex? + @supports_concurrent_reindex ||= begin + version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + version >= 12_000 + end + end + def deduplicate_and_reindex! deduplicate_preview_cards! - safety_assured { execute 'REINDEX INDEX CONCURRENTLY preview_cards_statuses_pkey' } + if supports_concurrent_reindex? + safety_assured { execute 'REINDEX INDEX CONCURRENTLY preview_cards_statuses_pkey' } + else + remove_index :preview_cards_statuses, name: :preview_cards_statuses_pkey + add_index :preview_cards_statuses, [:status_id, :preview_card_id], name: :preview_cards_statuses_pkey, algorithm: :concurrently, unique: true + end rescue ActiveRecord::RecordNotUnique retry end From ecd76fa413e31d4eb26e09fa4b65f8b13bbbb0b7 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 31 Aug 2023 17:21:06 +0200 Subject: [PATCH 08/16] Fix videos not playing in some browsers due to unsupported color space (#26745) --- app/models/media_attachment.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 88abd9339..984f4252a 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -100,6 +100,8 @@ class MediaAttachment < ApplicationRecord output: { 'loglevel' => 'fatal', 'preset' => 'veryfast', + 'movflags' => 'faststart', # Move metadata to start of file so playback can begin before download finishes + 'pix_fmt' => 'yuv420p', # Ensure color space for cross-browser compatibility 'c:v' => 'h264', 'c:a' => 'aac', 'b:a' => '192k', From 9bb2fb6b1484c90c5b2c6cc52ce148019e82a3e2 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 31 Aug 2023 19:04:27 +0200 Subject: [PATCH 09/16] Change importers to avoid a few inefficiencies (#26721) --- app/lib/importer/accounts_index_importer.rb | 6 ++-- app/lib/importer/base_importer.rb | 8 +++++ app/lib/importer/instances_index_importer.rb | 6 ++-- .../public_statuses_index_importer.rb | 6 ++-- app/lib/importer/statuses_index_importer.rb | 35 ++++++++----------- app/lib/importer/tags_index_importer.rb | 6 ++-- 6 files changed, 34 insertions(+), 33 deletions(-) diff --git a/app/lib/importer/accounts_index_importer.rb b/app/lib/importer/accounts_index_importer.rb index fd869c396..d8b919027 100644 --- a/app/lib/importer/accounts_index_importer.rb +++ b/app/lib/importer/accounts_index_importer.rb @@ -4,10 +4,10 @@ class Importer::AccountsIndexImporter < Importer::BaseImporter def import! scope.includes(:account_stat).find_in_batches(batch_size: @batch_size) do |tmp| in_work_unit(tmp) do |accounts| - bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: accounts).bulk_body + bulk = build_bulk_body(accounts) - indexed = bulk.count { |entry| entry[:index] } - deleted = bulk.count { |entry| entry[:delete] } + indexed = bulk.size + deleted = 0 Chewy::Index::Import::BulkRequest.new(index).perform(bulk) diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index cc1b7b44d..a21557d30 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -68,6 +68,14 @@ class Importer::BaseImporter protected + def build_bulk_body(to_import) + # Specialize `Chewy::Index::Import::BulkBuilder#bulk_body` to avoid a few + # inefficiencies, as none of our fields or join fields and we do not need + # `BulkBuilder`'s versatility. + crutches = Chewy::Index::Crutch::Crutches.new index, to_import + to_import.map { |object| { index: { _id: object.id, data: index.compose(object, crutches, fields: []) } } } + end + def in_work_unit(...) work_unit = Concurrent::Promises.future_on(@executor, ...) diff --git a/app/lib/importer/instances_index_importer.rb b/app/lib/importer/instances_index_importer.rb index 7318b51b5..ebdceb72e 100644 --- a/app/lib/importer/instances_index_importer.rb +++ b/app/lib/importer/instances_index_importer.rb @@ -4,10 +4,10 @@ class Importer::InstancesIndexImporter < Importer::BaseImporter def import! index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp| in_work_unit(tmp) do |instances| - bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: instances).bulk_body + bulk = build_bulk_body(instances) - indexed = bulk.count { |entry| entry[:index] } - deleted = bulk.count { |entry| entry[:delete] } + indexed = bulk.size + deleted = 0 Chewy::Index::Import::BulkRequest.new(index).perform(bulk) diff --git a/app/lib/importer/public_statuses_index_importer.rb b/app/lib/importer/public_statuses_index_importer.rb index 72d02318b..ebaac3794 100644 --- a/app/lib/importer/public_statuses_index_importer.rb +++ b/app/lib/importer/public_statuses_index_importer.rb @@ -5,11 +5,11 @@ class Importer::PublicStatusesIndexImporter < Importer::BaseImporter scope.select(:id).find_in_batches(batch_size: @batch_size) do |batch| in_work_unit(batch.pluck(:id)) do |status_ids| bulk = ActiveRecord::Base.connection_pool.with_connection do - Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll, :preview_cards).where(id: status_ids)).bulk_body + build_bulk_body(index.adapter.default_scope.where(id: status_ids)) end - indexed = bulk.count { |entry| entry[:index] } - deleted = bulk.count { |entry| entry[:delete] } + indexed = bulk.size + deleted = 0 Chewy::Index::Import::BulkRequest.new(index).perform(bulk) diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb index 285ddc871..08ad3e379 100644 --- a/app/lib/importer/statuses_index_importer.rb +++ b/app/lib/importer/statuses_index_importer.rb @@ -13,32 +13,25 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter scope.find_in_batches(batch_size: @batch_size) do |tmp| in_work_unit(tmp.map(&:status_id)) do |status_ids| - bulk = ActiveRecord::Base.connection_pool.with_connection do - Chewy::Index::Import::BulkBuilder.new(index, to_index: index.adapter.default_scope.where(id: status_ids)).bulk_body - end - - indexed = 0 deleted = 0 - # We can't use the delete_if proc to do the filtering because delete_if - # is called before rendering the data and we need to filter based - # on the results of the filter, so this filtering happens here instead - bulk.map! do |entry| - new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank? - { delete: entry[:index].except(:data) } - else - entry - end - - if new_entry[:index] - indexed += 1 - else - deleted += 1 + bulk = ActiveRecord::Base.connection_pool.with_connection do + to_index = index.adapter.default_scope.where(id: status_ids) + crutches = Chewy::Index::Crutch::Crutches.new index, to_index + to_index.map do |object| + # This is unlikely to happen, but the post may have been + # un-interacted with since it was queued for indexing + if object.searchable_by.empty? + deleted += 1 + { delete: { _id: object.id } } + else + { index: { _id: object.id, data: index.compose(object, crutches, fields: []) } } + end end - - new_entry end + indexed = bulk.size - deleted + Chewy::Index::Import::BulkRequest.new(index).perform(bulk) [indexed, deleted] diff --git a/app/lib/importer/tags_index_importer.rb b/app/lib/importer/tags_index_importer.rb index 77710ed7d..067fd8cd2 100644 --- a/app/lib/importer/tags_index_importer.rb +++ b/app/lib/importer/tags_index_importer.rb @@ -4,10 +4,10 @@ class Importer::TagsIndexImporter < Importer::BaseImporter def import! index.adapter.default_scope.find_in_batches(batch_size: @batch_size) do |tmp| in_work_unit(tmp) do |tags| - bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: tags).bulk_body + bulk = build_bulk_body(tags) - indexed = bulk.count { |entry| entry[:index] } - deleted = bulk.count { |entry| entry[:delete] } + indexed = bulk.size + deleted = 0 Chewy::Index::Import::BulkRequest.new(index).perform(bulk) From 00084581289b4b7afd120845363b16247c5fa93b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 31 Aug 2023 19:04:44 +0200 Subject: [PATCH 10/16] Fix search queries with slash causing or-condition (#26699) --- app/lib/search_query_transformer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index 86e3f5000..f10ccfb28 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -25,7 +25,7 @@ class SearchQueryTransformer < Parslet::Transform def clause_to_query(clause) case clause when TermClause - { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } } + { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'], operator: 'and' } } when PhraseClause { match_phrase: { text: { query: clause.phrase } } } else From 872145d1c2aec709770de10579d2a4f83c601f37 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 1 Sep 2023 09:27:03 +0200 Subject: [PATCH 11/16] Fix not being able to invoke phrase search using unicode quotation marks (#26687) --- app/services/search_service.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 40d82fc52..9a40d7bdd 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true class SearchService < BaseService + QUOTE_EQUIVALENT_CHARACTERS = /[“”„«»「」『』《》]/ + def call(query, account, limit, options = {}) - @query = query&.strip + @query = query&.strip&.gsub(QUOTE_EQUIVALENT_CHARACTERS, '"') @account = account @options = options @limit = limit.to_i From e754083e8a33778b5dd8d43efc5604ed50efb0e4 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 1 Sep 2023 09:43:12 +0200 Subject: [PATCH 12/16] Fix unmatched quotes and prefixes causing search to fail (#26701) --- app/lib/search_query_parser.rb | 4 +- app/lib/search_query_transformer.rb | 101 +++++++++++----------- spec/lib/search_query_parser_spec.rb | 98 +++++++++++++++++++++ spec/lib/search_query_transformer_spec.rb | 57 ++++++++++-- 4 files changed, 200 insertions(+), 60 deletions(-) create mode 100644 spec/lib/search_query_parser_spec.rb diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb index 5d6ffbf29..1c57b9b02 100644 --- a/app/lib/search_query_parser.rb +++ b/app/lib/search_query_parser.rb @@ -6,10 +6,10 @@ class SearchQueryParser < Parslet::Parser rule(:colon) { str(':') } rule(:space) { match('\s').repeat(1) } rule(:operator) { (str('+') | str('-')).as(:operator) } - rule(:prefix) { (term >> colon).as(:prefix) } + rule(:prefix) { term >> colon } rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) } rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) } - rule(:clause) { (operator.maybe >> prefix.maybe >> (phrase | term | shortcode)).as(:clause) } + rule(:clause) { (operator.maybe >> prefix.maybe.as(:prefix) >> (phrase | term | shortcode)).as(:clause) | prefix.as(:clause) | quote.as(:junk) } rule(:query) { (clause >> space.maybe).repeat.as(:query) } root(:query) end diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index f10ccfb28..e81c0c081 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -1,50 +1,32 @@ # frozen_string_literal: true class SearchQueryTransformer < Parslet::Transform + SUPPORTED_PREFIXES = %w( + has + is + language + from + before + after + during + ).freeze + class Query - attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses + attr_reader :must_not_clauses, :must_clauses, :filter_clauses def initialize(clauses) - grouped = clauses.chunk(&:operator).to_h - @should_clauses = grouped.fetch(:should, []) + grouped = clauses.compact.chunk(&:operator).to_h @must_not_clauses = grouped.fetch(:must_not, []) @must_clauses = grouped.fetch(:must, []) @filter_clauses = grouped.fetch(:filter, []) end def apply(search) - should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) } - must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) } - must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) } - filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) } + must_clauses.each { |clause| search = search.query.must(clause.to_query) } + must_not_clauses.each { |clause| search = search.query.must_not(clause.to_query) } + filter_clauses.each { |clause| search = search.filter(**clause.to_query) } search.query.minimum_should_match(1) end - - private - - def clause_to_query(clause) - case clause - when TermClause - { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'], operator: 'and' } } - when PhraseClause - { match_phrase: { text: { query: clause.phrase } } } - else - raise "Unexpected clause type: #{clause}" - end - end - - def clause_to_filter(clause) - case clause - when PrefixClause - if clause.negated? - { bool: { must_not: { clause.type => { clause.filter => clause.term } } } } - else - { clause.type => { clause.filter => clause.term } } - end - else - raise "Unexpected clause type: #{clause}" - end - end end class Operator @@ -63,31 +45,38 @@ class SearchQueryTransformer < Parslet::Transform end class TermClause - attr_reader :prefix, :operator, :term + attr_reader :operator, :term - def initialize(prefix, operator, term) - @prefix = prefix + def initialize(operator, term) @operator = Operator.symbol(operator) @term = term end + + def to_query + { multi_match: { type: 'most_fields', query: @term, fields: ['text', 'text.stemmed'], operator: 'and' } } + end end class PhraseClause - attr_reader :prefix, :operator, :phrase + attr_reader :operator, :phrase - def initialize(prefix, operator, phrase) - @prefix = prefix + def initialize(operator, phrase) @operator = Operator.symbol(operator) @phrase = phrase end + + def to_query + { match_phrase: { text: { query: @phrase } } } + end end class PrefixClause - attr_reader :type, :filter, :operator, :term + attr_reader :operator, :prefix, :term def initialize(prefix, operator, term, options = {}) - @negated = operator == '-' - @options = options + @prefix = prefix + @negated = operator == '-' + @options = options @operator = :filter case prefix @@ -116,12 +105,16 @@ class SearchQueryTransformer < Parslet::Transform @type = :range @term = { gte: term, lte: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' } else - raise Mastodon::SyntaxError + raise "Unknown prefix: #{prefix}" end end - def negated? - @negated + def to_query + if @negated + { bool: { must_not: { @type => { @filter => @term } } } } + else + { @type => { @filter => @term } } + end end private @@ -159,18 +152,26 @@ class SearchQueryTransformer < Parslet::Transform prefix = clause[:prefix][:term].to_s if clause[:prefix] operator = clause[:operator]&.to_s - if clause[:prefix] + if clause[:prefix] && SUPPORTED_PREFIXES.include?(prefix) PrefixClause.new(prefix, operator, clause[:term].to_s, current_account: current_account) + elsif clause[:prefix] + TermClause.new(operator, "#{prefix} #{clause[:term]}") elsif clause[:term] - TermClause.new(prefix, operator, clause[:term].to_s) + TermClause.new(operator, clause[:term].to_s) elsif clause[:shortcode] - TermClause.new(prefix, operator, ":#{clause[:term]}:") + TermClause.new(operator, ":#{clause[:term]}:") elsif clause[:phrase] - PhraseClause.new(prefix, operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s) + PhraseClause.new(operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s) else raise "Unexpected clause type: #{clause}" end end - rule(query: sequence(:clauses)) { Query.new(clauses) } + rule(junk: subtree(:junk)) do + nil + end + + rule(query: sequence(:clauses)) do + Query.new(clauses) + end end diff --git a/spec/lib/search_query_parser_spec.rb b/spec/lib/search_query_parser_spec.rb new file mode 100644 index 000000000..66b0e8f9e --- /dev/null +++ b/spec/lib/search_query_parser_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'parslet/rig/rspec' + +describe SearchQueryParser do + let(:parser) { described_class.new } + + context 'with term' do + it 'consumes "hello"' do + expect(parser.term).to parse('hello') + end + end + + context 'with prefix' do + it 'consumes "foo:"' do + expect(parser.prefix).to parse('foo:') + end + end + + context 'with operator' do + it 'consumes "+"' do + expect(parser.operator).to parse('+') + end + + it 'consumes "-"' do + expect(parser.operator).to parse('-') + end + end + + context 'with shortcode' do + it 'consumes ":foo:"' do + expect(parser.shortcode).to parse(':foo:') + end + end + + context 'with phrase' do + it 'consumes "hello world"' do + expect(parser.phrase).to parse('"hello world"') + end + end + + context 'with clause' do + it 'consumes "foo"' do + expect(parser.clause).to parse('foo') + end + + it 'consumes "-foo"' do + expect(parser.clause).to parse('-foo') + end + + it 'consumes "foo:bar"' do + expect(parser.clause).to parse('foo:bar') + end + + it 'consumes "-foo:bar"' do + expect(parser.clause).to parse('-foo:bar') + end + + it 'consumes \'foo:"hello world"\'' do + expect(parser.clause).to parse('foo:"hello world"') + end + + it 'consumes \'-foo:"hello world"\'' do + expect(parser.clause).to parse('-foo:"hello world"') + end + + it 'consumes "foo:"' do + expect(parser.clause).to parse('foo:') + end + + it 'consumes \'"\'' do + expect(parser.clause).to parse('"') + end + end + + context 'with query' do + it 'consumes "hello -world"' do + expect(parser.query).to parse('hello -world') + end + + it 'consumes \'foo "hello world"\'' do + expect(parser.query).to parse('foo "hello world"') + end + + it 'consumes "foo:bar hello"' do + expect(parser.query).to parse('foo:bar hello') + end + + it 'consumes \'"hello" world "\'' do + expect(parser.query).to parse('"hello" world "') + end + + it 'consumes "foo:bar bar: hello"' do + expect(parser.query).to parse('foo:bar bar: hello') + end + end +end diff --git a/spec/lib/search_query_transformer_spec.rb b/spec/lib/search_query_transformer_spec.rb index 953f9acb2..17f06d283 100644 --- a/spec/lib/search_query_transformer_spec.rb +++ b/spec/lib/search_query_transformer_spec.rb @@ -3,16 +3,57 @@ require 'rails_helper' describe SearchQueryTransformer do - describe 'initialization' do - let(:parser) { SearchQueryParser.new.parse('query') } + subject { described_class.new.apply(parser, current_account: nil) } - it 'sets attributes' do - transformer = described_class.new.apply(parser) + let(:parser) { SearchQueryParser.new.parse(query) } - expect(transformer.should_clauses.first).to be_nil - expect(transformer.must_clauses.first).to be_a(SearchQueryTransformer::TermClause) - expect(transformer.must_not_clauses.first).to be_nil - expect(transformer.filter_clauses.first).to be_nil + context 'with "hello world"' do + let(:query) { 'hello world' } + + it 'transforms clauses' do + expect(subject.must_clauses.map(&:term)).to match_array %w(hello world) + expect(subject.must_not_clauses).to be_empty + expect(subject.filter_clauses).to be_empty + end + end + + context 'with "hello -world"' do + let(:query) { 'hello -world' } + + it 'transforms clauses' do + expect(subject.must_clauses.map(&:term)).to match_array %w(hello) + expect(subject.must_not_clauses.map(&:term)).to match_array %w(world) + expect(subject.filter_clauses).to be_empty + end + end + + context 'with "hello is:reply"' do + let(:query) { 'hello is:reply' } + + it 'transforms clauses' do + expect(subject.must_clauses.map(&:term)).to match_array %w(hello) + expect(subject.must_not_clauses).to be_empty + expect(subject.filter_clauses.map(&:term)).to match_array %w(reply) + end + end + + context 'with "foo: bar"' do + let(:query) { 'foo: bar' } + + it 'transforms clauses' do + expect(subject.must_clauses.map(&:term)).to match_array %w(foo bar) + expect(subject.must_not_clauses).to be_empty + expect(subject.filter_clauses).to be_empty + end + end + + context 'with "foo:bar"' do + let(:query) { 'foo:bar' } + + it 'transforms clauses' do + expect(subject.must_clauses.map(&:term)).to contain_exactly('foo bar') + expect(subject.must_not_clauses).to be_empty + expect(subject.filter_clauses).to be_empty end end end From 630e558677d961e74ce82ab5e2a62a0760bbe815 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:17:08 +0200 Subject: [PATCH 13/16] Update dependency oj to v3.16.1 (#26749) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f1a61c5e0..6dd8c567f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -482,7 +482,7 @@ GEM nokogiri (1.15.4) mini_portile2 (~> 2.8.2) racc (~> 1.4) - oj (3.16.0) + oj (3.16.1) omniauth (2.1.1) hashie (>= 3.4.6) rack (>= 2.2.3) From bb0edb178f56a488017071e00a994c9033267ef9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:48:49 +0200 Subject: [PATCH 14/16] Update dependency pg to v1.5.4 (#26750) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6dd8c567f..4e30c4222 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -519,7 +519,7 @@ GEM parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - pg (1.5.3) + pg (1.5.4) pghero (3.3.3) activerecord (>= 6) posix-spawn (0.3.15) From 5c0a9aac3b2add81ff2a9c7e724dff4eb9c6451b Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 1 Sep 2023 15:09:44 +0200 Subject: [PATCH 15/16] Revert to using primary database in IndexingScheduler (#26754) --- app/workers/scheduler/indexing_scheduler.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index 1b09730c7..ff1b74444 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -16,9 +16,7 @@ class Scheduler::IndexingScheduler indexes.each do |type| with_redis do |redis| redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| - with_read_replica do - type.import!(ids) - end + type.import!(ids) redis.srem("chewy:queue:#{type.name}", ids) end From 6c4c72497a5722870e4432ef41dd4c9ec36a8928 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 1 Sep 2023 15:13:27 +0200 Subject: [PATCH 16/16] Fix search popout including full-text search instructions when full-text search is disabled (#26755) --- .../features/compose/components/search.jsx | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/javascript/mastodon/features/compose/components/search.jsx b/app/javascript/mastodon/features/compose/components/search.jsx index 1c629bcbb..848b81263 100644 --- a/app/javascript/mastodon/features/compose/components/search.jsx +++ b/app/javascript/mastodon/features/compose/components/search.jsx @@ -80,7 +80,7 @@ class Search extends PureComponent { handleKeyDown = (e) => { const { selectedOption } = this.state; - const options = this._getOptions().concat(this.defaultOptions); + const options = searchEnabled ? this._getOptions().concat(this.defaultOptions) : this._getOptions(); switch(e.key) { case 'Escape': @@ -353,15 +353,19 @@ class Search extends PureComponent { )} -

+ {searchEnabled && ( + <> +

-
- {this.defaultOptions.map(({ key, label, action }, i) => ( - - ))} -
+
+ {this.defaultOptions.map(({ key, label, action }, i) => ( + + ))} +
+ + )} );