From 91cc8d1e636a3515b15758d0ad449a0477ea8c4c Mon Sep 17 00:00:00 2001 From: Sumak <44816995+Kawsay@users.noreply.github.com> Date: Thu, 24 Feb 2022 14:46:21 +0100 Subject: [PATCH 01/24] fix #17601 change `:keys` to `:attribute_names` (#17637) --- spec/support/matchers/model/model_have_error_on_field.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/matchers/model/model_have_error_on_field.rb b/spec/support/matchers/model/model_have_error_on_field.rb index a5dfbf4570..85bdd82159 100644 --- a/spec/support/matchers/model/model_have_error_on_field.rb +++ b/spec/support/matchers/model/model_have_error_on_field.rb @@ -8,7 +8,7 @@ RSpec::Matchers.define :model_have_error_on_field do |expected| end failure_message do |record| - keys = record.errors.keys + keys = record.errors.attribute_names "expect record.errors(#{keys}) to include #{expected}" end From a29a982eaa0536a741b43ffb3397c74e3abe7196 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 24 Feb 2022 17:28:23 +0100 Subject: [PATCH 02/24] Change e-mail domain blocks to block IPs dynamically (#17635) * Change e-mail domain blocks to block IPs dynamically * Update app/workers/scheduler/email_domain_block_refresh_scheduler.rb Co-authored-by: Yamagishi Kazutoshi * Update app/workers/scheduler/email_domain_block_refresh_scheduler.rb Co-authored-by: Yamagishi Kazutoshi Co-authored-by: Yamagishi Kazutoshi --- .../admin/email_domain_blocks_controller.rb | 72 +++++++++++-------- app/models/email_domain_block.rb | 55 +++++++++----- app/models/form/email_domain_block_batch.rb | 30 ++++++++ app/models/status.rb | 1 + app/validators/blacklisted_email_validator.rb | 26 ++++--- app/validators/email_mx_validator.rb | 20 +++--- .../_email_domain_block.html.haml | 27 ++++--- .../admin/email_domain_blocks/index.html.haml | 28 +++++--- .../admin/email_domain_blocks/new.html.haml | 32 +++++++-- .../email_domain_block_refresh_scheduler.rb | 30 ++++++++ config/locales/en.yml | 13 +++- config/locales/simple_form.en.yml | 2 +- config/routes.rb | 7 +- config/sidekiq.yml | 4 ++ ...24010024_add_ips_to_email_domain_blocks.rb | 6 ++ db/schema.rb | 5 +- .../email_domain_blocks_controller_spec.rb | 42 +++++------ spec/models/email_domain_block_spec.rb | 27 +++++-- .../blacklisted_email_validator_spec.rb | 2 +- spec/validators/email_mx_validator_spec.rb | 56 ++++++++------- 20 files changed, 325 insertions(+), 160 deletions(-) create mode 100644 app/models/form/email_domain_block_batch.rb create mode 100644 app/workers/scheduler/email_domain_block_refresh_scheduler.rb create mode 100644 db/migrate/20220224010024_add_ips_to_email_domain_blocks.rb diff --git a/app/controllers/admin/email_domain_blocks_controller.rb b/app/controllers/admin/email_domain_blocks_controller.rb index f7bdfb0c5f..33ee079f36 100644 --- a/app/controllers/admin/email_domain_blocks_controller.rb +++ b/app/controllers/admin/email_domain_blocks_controller.rb @@ -6,7 +6,20 @@ module Admin def index authorize :email_domain_block, :index? + @email_domain_blocks = EmailDomainBlock.where(parent_id: nil).includes(:children).order(id: :desc).page(params[:page]) + @form = Form::EmailDomainBlockBatch.new + end + + def batch + @form = Form::EmailDomainBlockBatch.new(form_email_domain_block_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.email_domain_blocks.no_email_domain_block_selected') + rescue Mastodon::NotPermittedError + flash[:alert] = I18n.t('admin.custom_emojis.not_permitted') + ensure + redirect_to admin_email_domain_blocks_path end def new @@ -19,41 +32,25 @@ module Admin @email_domain_block = EmailDomainBlock.new(resource_params) - if @email_domain_block.save - log_action :create, @email_domain_block - - if @email_domain_block.with_dns_records? - hostnames = [] - ips = [] - - Resolv::DNS.open do |dns| - dns.timeouts = 5 + if action_from_button == 'save' + EmailDomainBlock.transaction do + @email_domain_block.save! + log_action :create, @email_domain_block - hostnames = dns.getresources(@email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s } - - ([@email_domain_block.domain] + hostnames).uniq.each do |hostname| - ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s }) - ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s }) - end - end - - (hostnames + ips).each do |hostname| - another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: @email_domain_block) - log_action :create, another_email_domain_block if another_email_domain_block.save + (@email_domain_block.other_domains || []).uniq.each do |domain| + other_email_domain_block = EmailDomainBlock.create!(domain: domain, parent: @email_domain_block) + log_action :create, other_email_domain_block end end redirect_to admin_email_domain_blocks_path, notice: I18n.t('admin.email_domain_blocks.created_msg') else + set_resolved_records render :new end - end - - def destroy - authorize @email_domain_block, :destroy? - @email_domain_block.destroy! - log_action :destroy, @email_domain_block - redirect_to admin_email_domain_blocks_path, notice: I18n.t('admin.email_domain_blocks.destroyed_msg') + rescue ActiveRecord::RecordInvalid + set_resolved_records + render :new end private @@ -62,8 +59,27 @@ module Admin @email_domain_block = EmailDomainBlock.find(params[:id]) end + def set_resolved_records + Resolv::DNS.open do |dns| + dns.timeouts = 5 + @resolved_records = dns.getresources(@email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a + end + end + def resource_params - params.require(:email_domain_block).permit(:domain, :with_dns_records) + params.require(:email_domain_block).permit(:domain, other_domains: []) + end + + def form_email_domain_block_batch_params + params.require(:form_email_domain_block_batch).permit(email_domain_block_ids: []) + end + + def action_from_button + if params[:delete] + 'delete' + elsif params[:save] + 'save' + end end end end diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb index f50fa46ba7..36e7e62ab8 100644 --- a/app/models/email_domain_block.rb +++ b/app/models/email_domain_block.rb @@ -3,11 +3,13 @@ # # Table name: email_domain_blocks # -# id :bigint(8) not null, primary key -# domain :string default(""), not null -# created_at :datetime not null -# updated_at :datetime not null -# parent_id :bigint(8) +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# parent_id :bigint(8) +# ips :inet is an Array +# last_refresh_at :datetime # class EmailDomainBlock < ApplicationRecord @@ -18,27 +20,42 @@ class EmailDomainBlock < ApplicationRecord validates :domain, presence: true, uniqueness: true, domain: true - def with_dns_records=(val) - @with_dns_records = ActiveModel::Type::Boolean.new.cast(val) - end + # Used for adding multiple blocks at once + attr_accessor :other_domains - def with_dns_records? - @with_dns_records + def history + @history ||= Trends::History.new('email_domain_blocks', id) end - alias with_dns_records with_dns_records? + def self.block?(domain_or_domains, ips: [], attempt_ip: nil) + domains = Array(domain_or_domains).map do |str| + domain = begin + if str.include?('@') + str.split('@', 2).last + else + str + end + end + + TagManager.instance.normalize_domain(domain) if domain.present? + rescue Addressable::URI::InvalidURIError + nil + end - def self.block?(email) - _, domain = email.split('@', 2) + # If some of the inputs passed in are invalid, we definitely want to + # block the attempt, but we also want to register hits against any + # other valid matches - return true if domain.nil? + blocked = domains.any?(&:nil?) - begin - domain = TagManager.instance.normalize_domain(domain) - rescue Addressable::URI::InvalidURIError - return true + scope = where(domain: domains) + scope = scope.or(where('ips && ARRAY[?]::inet[]', ips)) if ips.any? + + scope.find_each do |block| + blocked = true + block.history.add(attempt_ip) if attempt_ip.present? end - where(domain: domain).exists? + blocked end end diff --git a/app/models/form/email_domain_block_batch.rb b/app/models/form/email_domain_block_batch.rb new file mode 100644 index 0000000000..df120182bc --- /dev/null +++ b/app/models/form/email_domain_block_batch.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Form::EmailDomainBlockBatch + include ActiveModel::Model + include Authorization + include AccountableConcern + + attr_accessor :email_domain_block_ids, :action, :current_account + + def save + case action + when 'delete' + delete! + end + end + + private + + def email_domain_blocks + @email_domain_blocks ||= EmailDomainBlock.where(id: email_domain_block_ids) + end + + def delete! + email_domain_blocks.each do |email_domain_block| + authorize(email_domain_block, :destroy?) + email_domain_block.destroy! + log_action :destroy, email_domain_block + end + end +end diff --git a/app/models/status.rb b/app/models/status.rb index 2e3df98a1f..96e41b1d32 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -24,6 +24,7 @@ # poll_id :bigint(8) # deleted_at :datetime # edited_at :datetime +# trendable :boolean # class Status < ApplicationRecord diff --git a/app/validators/blacklisted_email_validator.rb b/app/validators/blacklisted_email_validator.rb index eb66ad93d7..9b3f2e33e5 100644 --- a/app/validators/blacklisted_email_validator.rb +++ b/app/validators/blacklisted_email_validator.rb @@ -4,41 +4,39 @@ class BlacklistedEmailValidator < ActiveModel::Validator def validate(user) return if user.valid_invitation? || user.email.blank? - @email = user.email - - user.errors.add(:email, :blocked) if blocked_email_provider? - user.errors.add(:email, :taken) if blocked_canonical_email? + user.errors.add(:email, :blocked) if blocked_email_provider?(user.email, user.sign_up_ip) + user.errors.add(:email, :taken) if blocked_canonical_email?(user.email) end private - def blocked_email_provider? - disallowed_through_email_domain_block? || disallowed_through_configuration? || not_allowed_through_configuration? + def blocked_email_provider?(email, ip) + disallowed_through_email_domain_block?(email, ip) || disallowed_through_configuration?(email) || not_allowed_through_configuration?(email) end - def blocked_canonical_email? - CanonicalEmailBlock.block?(@email) + def blocked_canonical_email?(email) + CanonicalEmailBlock.block?(email) end - def disallowed_through_email_domain_block? - EmailDomainBlock.block?(@email) + def disallowed_through_email_domain_block?(email, ip) + EmailDomainBlock.block?(email, attempt_ip: ip) end - def not_allowed_through_configuration? + def not_allowed_through_configuration?(email) return false if Rails.configuration.x.email_domains_whitelist.blank? domains = Rails.configuration.x.email_domains_whitelist.gsub('.', '\.') regexp = Regexp.new("@(.+\\.)?(#{domains})$", true) - @email !~ regexp + email !~ regexp end - def disallowed_through_configuration? + def disallowed_through_configuration?(email) return false if Rails.configuration.x.email_domains_blacklist.blank? domains = Rails.configuration.x.email_domains_blacklist.gsub('.', '\.') regexp = Regexp.new("@(.+\\.)?(#{domains})", true) - regexp.match?(@email) + regexp.match?(email) end end diff --git a/app/validators/email_mx_validator.rb b/app/validators/email_mx_validator.rb index dceef50290..237ca4c7bc 100644 --- a/app/validators/email_mx_validator.rb +++ b/app/validators/email_mx_validator.rb @@ -11,11 +11,11 @@ class EmailMxValidator < ActiveModel::Validator if domain.blank? user.errors.add(:email, :invalid) elsif !on_allowlist?(domain) - ips, hostnames = resolve_mx(domain) + resolved_ips, resolved_domains = resolve_mx(domain) - if ips.empty? + if resolved_ips.empty? user.errors.add(:email, :unreachable) - elsif on_blacklist?(hostnames + ips) + elsif on_blacklist?(resolved_domains, resolved_ips, user.sign_up_ip) user.errors.add(:email, :blocked) end end @@ -40,24 +40,24 @@ class EmailMxValidator < ActiveModel::Validator end def resolve_mx(domain) - hostnames = [] - ips = [] + records = [] + ips = [] Resolv::DNS.open do |dns| dns.timeouts = 5 - hostnames = dns.getresources(domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s } + records = dns.getresources(domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s } - ([domain] + hostnames).uniq.each do |hostname| + ([domain] + records).uniq.each do |hostname| ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s }) ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s }) end end - [ips, hostnames] + [ips, records] end - def on_blacklist?(values) - EmailDomainBlock.where(domain: values.uniq).any? + def on_blacklist?(domains, resolved_ips, attempt_ip) + EmailDomainBlock.block?(domains, ips: resolved_ips, attempt_ip: attempt_ip) end end diff --git a/app/views/admin/email_domain_blocks/_email_domain_block.html.haml b/app/views/admin/email_domain_blocks/_email_domain_block.html.haml index 41ab8c1712..c5a55bc27c 100644 --- a/app/views/admin/email_domain_blocks/_email_domain_block.html.haml +++ b/app/views/admin/email_domain_blocks/_email_domain_block.html.haml @@ -1,15 +1,14 @@ -%tr - %td - %samp= email_domain_block.domain - %td - = table_link_to 'trash', t('admin.email_domain_blocks.delete'), admin_email_domain_block_path(email_domain_block), method: :delete +.batch-table__row + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :email_domain_block_ids, { multiple: true, include_hidden: false }, email_domain_block.id + .batch-table__row__content.pending-account + .pending-account__header + %samp= link_to email_domain_block.domain, admin_accounts_path(email: "%@#{email_domain_block.domain}") -- email_domain_block.children.each do |child_email_domain_block| - %tr - %td - %samp= child_email_domain_block.domain - %span.muted-hint - = surround '(', ')' do - = t('admin.email_domain_blocks.from_html', domain: content_tag(:samp, email_domain_block.domain)) - %td - = table_link_to 'trash', t('admin.email_domain_blocks.delete'), admin_email_domain_block_path(child_email_domain_block), method: :delete + %br/ + + - if email_domain_block.parent.present? + = t('admin.email_domain_blocks.resolved_through_html', domain: content_tag(:samp, email_domain_block.parent.domain)) + • + + = t('admin.email_domain_blocks.attempts_over_week', count: email_domain_block.history.reduce(0) { |sum, day| sum + day.accounts }) diff --git a/app/views/admin/email_domain_blocks/index.html.haml b/app/views/admin/email_domain_blocks/index.html.haml index fa5d86b67a..b073e87162 100644 --- a/app/views/admin/email_domain_blocks/index.html.haml +++ b/app/views/admin/email_domain_blocks/index.html.haml @@ -4,16 +4,22 @@ - content_for :heading_actions do = link_to t('admin.email_domain_blocks.add_new'), new_admin_email_domain_block_path, class: 'button' -- if @email_domain_blocks.empty? - %div.muted-hint.center-text=t 'admin.email_domain_blocks.empty' -- else - .table-wrapper - %table.table - %thead - %tr - %th= t('admin.email_domain_blocks.domain') - %th - %tbody - = render partial: 'email_domain_block', collection: @email_domain_blocks +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + += form_for(@form, url: batch_admin_email_domain_blocks_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('times'), t('admin.email_domain_blocks.delete')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @email_domain_blocks.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'email_domain_block', collection: @email_domain_blocks.flat_map { |x| [x, x.children.to_a].flatten }, locals: { f: f } = paginate @email_domain_blocks diff --git a/app/views/admin/email_domain_blocks/new.html.haml b/app/views/admin/email_domain_blocks/new.html.haml index 4a346f2406..524b699688 100644 --- a/app/views/admin/email_domain_blocks/new.html.haml +++ b/app/views/admin/email_domain_blocks/new.html.haml @@ -1,14 +1,38 @@ - content_for :page_title do = t('.title') +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + = simple_form_for @email_domain_block, url: admin_email_domain_blocks_path do |f| = render 'shared/error_messages', object: @email_domain_block .fields-group - = f.input :domain, wrapper: :with_block_label, label: t('admin.email_domain_blocks.domain') + = f.input :domain, wrapper: :with_block_label, label: t('admin.email_domain_blocks.domain'), input_html: { readonly: defined?(@resolved_records) } - .fields-group - = f.input :with_dns_records, as: :boolean, wrapper: :with_label + - if defined?(@resolved_records) + %p.hint= t('admin.email_domain_blocks.resolved_dns_records_hint_html') + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + .batch-table__body + - @resolved_records.each do |record| + .batch-table__row + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.input_field :other_domains, as: :boolean, checked_value: record.exchange.to_s, include_hidden: false, multiple: true + .batch-table__row__content.pending-account + .pending-account__header + %samp= record.exchange.to_s + %br + = t('admin.email_domain_blocks.dns.types.mx') + + %hr.spacer/ .actions - = f.button :button, t('.create'), type: :submit + - if defined?(@resolved_records) + = f.button :button, t('.create'), type: :submit, name: :save + - else + = f.button :button, t('.resolve'), type: :submit, name: :resolve diff --git a/app/workers/scheduler/email_domain_block_refresh_scheduler.rb b/app/workers/scheduler/email_domain_block_refresh_scheduler.rb new file mode 100644 index 0000000000..c67be68438 --- /dev/null +++ b/app/workers/scheduler/email_domain_block_refresh_scheduler.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Scheduler::EmailDomainBlockRefreshScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + def perform + Resolv::DNS.open do |dns| + dns.timeouts = 5 + + EmailDomainBlock.find_each do |email_domain_block| + ips = begin + if ip?(email_domain_block.domain) + [email_domain_block.domain] + else + dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::A).to_a + dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::AAAA).to_a.map { |resource| resource.address.to_s } + end + end + + email_domain_block.update(ips: ips, last_refresh_at: Time.now.utc) + end + end + end + + def ip?(str) + str =~ Regexp.union([Resolv::IPv4::Regex, Resolv::IPv6::Regex]) + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index c206c893b7..f045174a9b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -467,15 +467,22 @@ en: view: View domain block email_domain_blocks: add_new: Add new + attempts_over_week: + one: "%{count} attempt over the last week" + other: "%{count} sign-up attempts over the last week" created_msg: Successfully blocked e-mail domain delete: Delete - destroyed_msg: Successfully unblocked e-mail domain + dns: + types: + mx: MX record domain: Domain - empty: No e-mail domains currently blocked. - from_html: from %{domain} new: create: Add domain + resolve: Resolve domain title: Block new e-mail domain + no_email_domain_block_selected: No e-mail domain blocks were changed as none were selected + resolved_dns_records_hint_html: The domain name resolves to the following MX domains, which are ultimately responsible for accepting e-mail. Blocking an MX domain will block sign-ups from any e-mail address which uses the same MX domain, even if the visible domain name is different. Be careful not to block major e-mail providers. + resolved_through_html: Resolved through %{domain} title: Blocked e-mail domains follow_recommendations: description_html: "Follow recommendations help new users quickly find interesting content. When a user has not interacted with others enough to form personalized follow recommendations, these accounts are recommended instead. They are re-calculated on a daily basis from a mix of accounts with the highest recent engagements and highest local follower counts for a given language." diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 03eefd0d5a..c5e75b408c 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -64,7 +64,7 @@ en: domain_allow: domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored email_domain_block: - domain: This can be the domain name that shows up in the e-mail address, the MX record that domain resolves to, or IP of the server that MX record resolves to. Those will be checked upon user sign-up and the sign-up will be rejected. + domain: This can be the domain name that shows up in the e-mail address or the MX record it uses. They will be checked upon sign-up. with_dns_records: An attempt to resolve the given domain's DNS records will be made and the results will also be blocked featured_tag: name: 'You might want to use one of these:' diff --git a/config/routes.rb b/config/routes.rb index f59f2a5bba..176438e452 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -193,7 +193,12 @@ Rails.application.routes.draw do resources :domain_allows, only: [:new, :create, :show, :destroy] resources :domain_blocks, only: [:new, :create, :show, :destroy, :update, :edit] - resources :email_domain_blocks, only: [:index, :new, :create, :destroy] + resources :email_domain_blocks, only: [:index, :new, :create] do + collection do + post :batch + end + end + resources :action_logs, only: [:index] resources :warning_presets, except: [:new] diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 9dde5a053b..c8b1a20dd3 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -17,6 +17,10 @@ every: '5m' class: Scheduler::Trends::RefreshScheduler queue: scheduler + email_domain_block_refresh_scheduler: + every: '1h' + class: Scheduler::EmailDomainBlockRefreshScheduler + queue: scheduler trends_review_notifications_scheduler: every: '2h' class: Scheduler::Trends::ReviewNotificationsScheduler diff --git a/db/migrate/20220224010024_add_ips_to_email_domain_blocks.rb b/db/migrate/20220224010024_add_ips_to_email_domain_blocks.rb new file mode 100644 index 0000000000..1b19a2aa1f --- /dev/null +++ b/db/migrate/20220224010024_add_ips_to_email_domain_blocks.rb @@ -0,0 +1,6 @@ +class AddIpsToEmailDomainBlocks < ActiveRecord::Migration[6.1] + def change + add_column :email_domain_blocks, :ips, :inet, array: true + add_column :email_domain_blocks, :last_refresh_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index 8842dcd8c0..0e9b6e6191 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_02_10_153119) do +ActiveRecord::Schema.define(version: 2022_02_24_010024) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -387,6 +387,9 @@ ActiveRecord::Schema.define(version: 2022_02_10_153119) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.bigint "parent_id" + t.inet "ips", array: true + t.datetime "last_refresh_at" + t.index ["domain"], name: "index_email_domain_blocks_on_domain", unique: true end diff --git a/spec/controllers/admin/email_domain_blocks_controller_spec.rb b/spec/controllers/admin/email_domain_blocks_controller_spec.rb index 52db56f4e8..cf194579da 100644 --- a/spec/controllers/admin/email_domain_blocks_controller_spec.rb +++ b/spec/controllers/admin/email_domain_blocks_controller_spec.rb @@ -17,43 +17,43 @@ RSpec.describe Admin::EmailDomainBlocksController, type: :controller do EmailDomainBlock.paginates_per default_per_page end - it 'renders email blacks' do + it 'returns http success' do 2.times { Fabricate(:email_domain_block) } - get :index, params: { page: 2 } - - assigned = assigns(:email_domain_blocks) - expect(assigned.count).to eq 1 - expect(assigned.klass).to be EmailDomainBlock expect(response).to have_http_status(200) end end describe 'GET #new' do - it 'assigns a new email black' do + it 'returns http success' do get :new - - expect(assigns(:email_domain_block)).to be_instance_of(EmailDomainBlock) expect(response).to have_http_status(200) end end describe 'POST #create' do - it 'blocks the domain when succeeded to save' do - post :create, params: { email_domain_block: { domain: 'example.com' } } - - expect(flash[:notice]).to eq I18n.t('admin.email_domain_blocks.created_msg') - expect(response).to redirect_to(admin_email_domain_blocks_path) + context 'when resolve button is pressed' do + before do + post :create, params: { email_domain_block: { domain: 'example.com' } } + end + + it 'renders new template' do + expect(response).to render_template(:new) + end end - end - describe 'DELETE #destroy' do - it 'unblocks the domain' do - email_domain_block = Fabricate(:email_domain_block) - delete :destroy, params: { id: email_domain_block.id } + context 'when save button is pressed' do + before do + post :create, params: { email_domain_block: { domain: 'example.com' }, save: '' } + end + + it 'blocks the domain' do + expect(EmailDomainBlock.find_by(domain: 'example.com')).to_not be_nil + end - expect(flash[:notice]).to eq I18n.t('admin.email_domain_blocks.destroyed_msg') - expect(response).to redirect_to(admin_email_domain_blocks_path) + it 'redirects to e-mail domain blocks' do + expect(response).to redirect_to(admin_email_domain_blocks_path) + end end end end diff --git a/spec/models/email_domain_block_spec.rb b/spec/models/email_domain_block_spec.rb index efd2853a96..567a32c328 100644 --- a/spec/models/email_domain_block_spec.rb +++ b/spec/models/email_domain_block_spec.rb @@ -9,14 +9,29 @@ RSpec.describe EmailDomainBlock, type: :model do end describe 'block?' do - it 'returns true if the domain is registed' do - Fabricate(:email_domain_block, domain: 'example.com') - expect(EmailDomainBlock.block?('nyarn@example.com')).to eq true + let(:input) { nil } + + context 'given an e-mail address' do + let(:input) { 'nyarn@example.com' } + + it 'returns true if the domain is blocked' do + Fabricate(:email_domain_block, domain: 'example.com') + expect(EmailDomainBlock.block?(input)).to be true + end + + it 'returns false if the domain is not blocked' do + Fabricate(:email_domain_block, domain: 'other-example.com') + expect(EmailDomainBlock.block?(input)).to be false + end end - it 'returns true if the domain is not registed' do - Fabricate(:email_domain_block, domain: 'example.com') - expect(EmailDomainBlock.block?('nyarn@example.net')).to eq false + context 'given an array of domains' do + let(:input) { %w(foo.com mail.foo.com) } + + it 'returns true if the domain is blocked' do + Fabricate(:email_domain_block, domain: 'mail.foo.com') + expect(EmailDomainBlock.block?(input)).to be true + end end end end diff --git a/spec/validators/blacklisted_email_validator_spec.rb b/spec/validators/blacklisted_email_validator_spec.rb index f7d5e01bc8..351de07076 100644 --- a/spec/validators/blacklisted_email_validator_spec.rb +++ b/spec/validators/blacklisted_email_validator_spec.rb @@ -4,7 +4,7 @@ require 'rails_helper' RSpec.describe BlacklistedEmailValidator, type: :validator do describe '#validate' do - let(:user) { double(email: 'info@mail.com', errors: errors) } + let(:user) { double(email: 'info@mail.com', sign_up_ip: '1.2.3.4', errors: errors) } let(:errors) { double(add: nil) } before do diff --git a/spec/validators/email_mx_validator_spec.rb b/spec/validators/email_mx_validator_spec.rb index 550e91996b..af0eb98f5c 100644 --- a/spec/validators/email_mx_validator_spec.rb +++ b/spec/validators/email_mx_validator_spec.rb @@ -4,24 +4,28 @@ require 'rails_helper' describe EmailMxValidator do describe '#validate' do - let(:user) { double(email: 'foo@example.com', errors: double(add: nil)) } - - it 'does not add errors if there are no DNS records for an e-mail domain that is explicitly allowed' do - old_whitelist = Rails.configuration.x.email_domains_whitelist - Rails.configuration.x.email_domains_whitelist = 'example.com' - - resolver = double - - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) - allow(resolver).to receive(:timeouts=).and_return(nil) - allow(Resolv::DNS).to receive(:open).and_yield(resolver) - - subject.validate(user) - expect(user.errors).to_not have_received(:add) - - Rails.configuration.x.email_domains_whitelist = old_whitelist + let(:user) { double(email: 'foo@example.com', sign_up_ip: '1.2.3.4', errors: double(add: nil)) } + + context 'for an e-mail domain that is explicitly allowed' do + around do |block| + tmp = Rails.configuration.x.email_domains_whitelist + Rails.configuration.x.email_domains_whitelist = 'example.com' + block.call + Rails.configuration.x.email_domains_whitelist = tmp + end + + it 'does not add errors if there are no DNS records' do + resolver = double + + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:timeouts=).and_return(nil) + allow(Resolv::DNS).to receive(:open).and_yield(resolver) + + subject.validate(user) + expect(user.errors).to_not have_received(:add) + end end it 'adds an error if there are no DNS records for the e-mail domain' do @@ -37,7 +41,7 @@ describe EmailMxValidator do expect(user.errors).to have_received(:add) end - it 'adds an error if a MX record exists but does not lead to an IP' do + it 'adds an error if a MX record does not lead to an IP' do resolver = double allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) @@ -53,7 +57,7 @@ describe EmailMxValidator do end it 'adds an error if the A record is blacklisted' do - EmailDomainBlock.create!(domain: '1.2.3.4') + EmailDomainBlock.create!(domain: 'alternate-example.com', ips: ['1.2.3.4']) resolver = double allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) @@ -67,7 +71,7 @@ describe EmailMxValidator do end it 'adds an error if the AAAA record is blacklisted' do - EmailDomainBlock.create!(domain: 'fd00::1') + EmailDomainBlock.create!(domain: 'alternate-example.com', ips: ['fd00::1']) resolver = double allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) @@ -80,8 +84,8 @@ describe EmailMxValidator do expect(user.errors).to have_received(:add) end - it 'adds an error if the MX record is blacklisted' do - EmailDomainBlock.create!(domain: '2.3.4.5') + it 'adds an error if the A record of the MX record is blacklisted' do + EmailDomainBlock.create!(domain: 'mail.other-domain.com', ips: ['2.3.4.5']) resolver = double allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) @@ -96,8 +100,8 @@ describe EmailMxValidator do expect(user.errors).to have_received(:add) end - it 'adds an error if the MX IPv6 record is blacklisted' do - EmailDomainBlock.create!(domain: 'fd00::2') + it 'adds an error if the AAAA record of the MX record is blacklisted' do + EmailDomainBlock.create!(domain: 'mail.other-domain.com', ips: ['fd00::2']) resolver = double allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) @@ -112,7 +116,7 @@ describe EmailMxValidator do expect(user.errors).to have_received(:add) end - it 'adds an error if the MX hostname is blacklisted' do + it 'adds an error if the MX record is blacklisted' do EmailDomainBlock.create!(domain: 'mail.example.com') resolver = double From 27965ce5edff20db2de1dd233c88f8393bb0da0b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 25 Feb 2022 00:34:14 +0100 Subject: [PATCH 03/24] Add trending statuses (#17431) * Add trending statuses * Fix dangling items with stale scores in localized sets * Various fixes and improvements - Change approve_all/reject_all to approve_accounts/reject_accounts - Change Trends::Query methods to not mutate the original query - Change Trends::Query#skip to offset - Change follow recommendations to be refreshed in a transaction * Add tests for trending statuses filtering behaviour * Fix not applying filtering scope in controller --- .rubocop.yml | 5 +- .../preview_card_providers_controller.rb | 12 +- .../admin/trends/links_controller.rb | 20 +-- .../admin/trends/statuses_controller.rb | 45 ++++++ .../admin/trends/tags_controller.rb | 12 +- .../api/v1/admin/trends/links_controller.rb | 19 +++ .../v1/admin/trends/statuses_controller.rb | 19 +++ .../api/v1/admin/trends/tags_controller.rb | 2 +- .../api/v1/trends/links_controller.rb | 6 +- .../api/v1/trends/statuses_controller.rb | 27 ++++ .../api/v1/trends/tags_controller.rb | 2 +- app/controllers/concerns/localized.rb | 4 + app/helpers/admin/filter_helper.rb | 7 +- app/helpers/languages_helper.rb | 2 +- app/javascript/styles/mastodon/accounts.scss | 10 +- app/javascript/styles/mastodon/tables.scss | 7 + app/lib/activitypub/activity/announce.rb | 3 +- app/lib/activitypub/activity/like.rb | 2 + app/mailers/admin_mailer.rb | 27 ++-- app/models/account.rb | 30 ++-- app/models/status.rb | 12 ++ app/models/trends.rb | 26 +++- app/models/trends/base.rb | 20 ++- app/models/trends/links.rb | 52 ++++--- .../{form => trends}/preview_card_batch.rb | 22 +-- .../{ => trends}/preview_card_filter.rb | 23 +-- .../preview_card_provider_batch.rb | 6 +- .../preview_card_provider_filter.rb | 2 +- app/models/trends/query.rb | 106 +++++++++++++ app/models/trends/status_batch.rb | 65 ++++++++ app/models/trends/status_filter.rb | 46 ++++++ app/models/trends/statuses.rb | 142 ++++++++++++++++++ app/models/{form => trends}/tag_batch.rb | 6 +- app/models/{ => trends}/tag_filter.rb | 10 +- app/models/trends/tags.rb | 36 ++--- app/models/user.rb | 2 +- app/policies/account_policy.rb | 4 + app/policies/preview_card_policy.rb | 2 +- app/policies/preview_card_provider_policy.rb | 2 +- app/policies/status_policy.rb | 4 + app/policies/tag_policy.rb | 4 + app/services/delete_account_service.rb | 32 ++-- app/services/favourite_service.rb | 2 + app/services/reblog_service.rb | 3 +- .../custom_emojis/_custom_emoji.html.haml | 2 +- .../follow_recommendations/show.html.haml | 6 +- app/views/admin/trends/links/index.html.haml | 34 +++-- .../preview_card_providers/index.html.haml | 2 +- .../admin/trends/statuses/_status.html.haml | 30 ++++ .../admin/trends/statuses/index.html.haml | 43 ++++++ app/views/admin/trends/tags/index.html.haml | 4 +- .../admin_mailer/_new_trending_links.text.erb | 14 ++ .../_new_trending_statuses.text.erb | 14 ++ .../admin_mailer/_new_trending_tags.text.erb | 14 ++ .../admin_mailer/new_trending_links.text.erb | 16 -- .../admin_mailer/new_trending_tags.text.erb | 16 -- app/views/admin_mailer/new_trends.text.erb | 13 ++ app/views/application/_sidebar.html.haml | 2 +- .../follow_recommendations_scheduler.rb | 8 +- config/brakeman.ignore | 68 +++------ config/locales/en.yml | 34 +++-- config/navigation.rb | 1 + config/routes.rb | 9 ++ ...0220202200743_add_trendable_to_accounts.rb | 7 + ...0220202200926_add_trendable_to_statuses.rb | 5 + ...201015_remove_trust_level_from_accounts.rb | 9 ++ db/schema.rb | 6 +- .../api/v1/trends/tags_controller_spec.rb | 7 +- spec/mailers/previews/admin_mailer_preview.rb | 11 +- spec/models/trends/statuses_spec.rb | 110 ++++++++++++++ spec/models/trends/tags_spec.rb | 6 +- 71 files changed, 1074 insertions(+), 307 deletions(-) create mode 100644 app/controllers/admin/trends/statuses_controller.rb create mode 100644 app/controllers/api/v1/admin/trends/links_controller.rb create mode 100644 app/controllers/api/v1/admin/trends/statuses_controller.rb create mode 100644 app/controllers/api/v1/trends/statuses_controller.rb rename app/models/{form => trends}/preview_card_batch.rb (82%) rename app/models/{ => trends}/preview_card_filter.rb (52%) rename app/models/{form => trends}/preview_card_provider_batch.rb (91%) rename app/models/{ => trends}/preview_card_provider_filter.rb (95%) create mode 100644 app/models/trends/query.rb create mode 100644 app/models/trends/status_batch.rb create mode 100644 app/models/trends/status_filter.rb create mode 100644 app/models/trends/statuses.rb rename app/models/{form => trends}/tag_batch.rb (81%) rename app/models/{ => trends}/tag_filter.rb (76%) create mode 100644 app/views/admin/trends/statuses/_status.html.haml create mode 100644 app/views/admin/trends/statuses/index.html.haml create mode 100644 app/views/admin_mailer/_new_trending_links.text.erb create mode 100644 app/views/admin_mailer/_new_trending_statuses.text.erb create mode 100644 app/views/admin_mailer/_new_trending_tags.text.erb delete mode 100644 app/views/admin_mailer/new_trending_links.text.erb delete mode 100644 app/views/admin_mailer/new_trending_tags.text.erb create mode 100644 app/views/admin_mailer/new_trends.text.erb create mode 100644 db/migrate/20220202200743_add_trendable_to_accounts.rb create mode 100644 db/migrate/20220202200926_add_trendable_to_statuses.rb create mode 100644 db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb create mode 100644 spec/models/trends/statuses_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 2af0f59bbe..68634e9e33 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -32,10 +32,11 @@ Layout/EmptyLineAfterGuardClause: Layout/EmptyLinesAroundAttributeAccessor: Enabled: true +Layout/FirstHashElementIndentation: + EnforcedStyle: consistent + Layout/HashAlignment: Enabled: false - # EnforcedHashRocketStyle: table - # EnforcedColonStyle: table Layout/SpaceAroundMethodCallOperator: Enabled: true diff --git a/app/controllers/admin/trends/links/preview_card_providers_controller.rb b/app/controllers/admin/trends/links/preview_card_providers_controller.rb index 2c26e03f36..40a466cd63 100644 --- a/app/controllers/admin/trends/links/preview_card_providers_controller.rb +++ b/app/controllers/admin/trends/links/preview_card_providers_controller.rb @@ -5,11 +5,11 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll authorize :preview_card_provider, :index? @preview_card_providers = filtered_preview_card_providers.page(params[:page]) - @form = Form::PreviewCardProviderBatch.new + @form = Trends::PreviewCardProviderBatch.new end def batch - @form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) + @form = Trends::PreviewCardProviderBatch.new(trends_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') @@ -20,15 +20,15 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll private def filtered_preview_card_providers - PreviewCardProviderFilter.new(filter_params).results + Trends::PreviewCardProviderFilter.new(filter_params).results end def filter_params - params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS) + params.slice(:page, *Trends::PreviewCardProviderFilter::KEYS).permit(:page, *Trends::PreviewCardProviderFilter::KEYS) end - def form_preview_card_provider_batch_params - params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) + def trends_preview_card_provider_batch_params + params.require(:trends_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) end def action_from_button diff --git a/app/controllers/admin/trends/links_controller.rb b/app/controllers/admin/trends/links_controller.rb index 619b37deb1..434eec5fee 100644 --- a/app/controllers/admin/trends/links_controller.rb +++ b/app/controllers/admin/trends/links_controller.rb @@ -5,11 +5,11 @@ class Admin::Trends::LinksController < Admin::BaseController authorize :preview_card, :index? @preview_cards = filtered_preview_cards.page(params[:page]) - @form = Form::PreviewCardBatch.new + @form = Trends::PreviewCardBatch.new end def batch - @form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) + @form = Trends::PreviewCardBatch.new(trends_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') @@ -20,26 +20,26 @@ class Admin::Trends::LinksController < Admin::BaseController private def filtered_preview_cards - PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results + Trends::PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results end def filter_params - params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS) + params.slice(:page, *Trends::PreviewCardFilter::KEYS).permit(:page, *Trends::PreviewCardFilter::KEYS) end - def form_preview_card_batch_params - params.require(:form_preview_card_batch).permit(:action, preview_card_ids: []) + def trends_preview_card_batch_params + params.require(:trends_preview_card_batch).permit(:action, preview_card_ids: []) end def action_from_button if params[:approve] 'approve' - elsif params[:approve_all] - 'approve_all' + elsif params[:approve_providers] + 'approve_providers' elsif params[:reject] 'reject' - elsif params[:reject_all] - 'reject_all' + elsif params[:reject_providers] + 'reject_providers' end end end diff --git a/app/controllers/admin/trends/statuses_controller.rb b/app/controllers/admin/trends/statuses_controller.rb new file mode 100644 index 0000000000..7662427387 --- /dev/null +++ b/app/controllers/admin/trends/statuses_controller.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class Admin::Trends::StatusesController < Admin::BaseController + def index + authorize :status, :index? + + @statuses = filtered_statuses.page(params[:page]) + @form = Trends::StatusBatch.new + end + + def batch + @form = Trends::StatusBatch.new(trends_status_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_statuses_path(filter_params) + end + + private + + def filtered_statuses + Trends::StatusFilter.new(filter_params.with_defaults(trending: 'all')).results.includes(:account, :media_attachments, :active_mentions) + end + + def filter_params + params.slice(:page, *Trends::StatusFilter::KEYS).permit(:page, *Trends::StatusFilter::KEYS) + end + + def trends_status_batch_params + params.require(:trends_status_batch).permit(:action, status_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:approve_accounts] + 'approve_accounts' + elsif params[:reject] + 'reject' + elsif params[:reject_accounts] + 'reject_accounts' + end + end +end diff --git a/app/controllers/admin/trends/tags_controller.rb b/app/controllers/admin/trends/tags_controller.rb index 91ff33d406..f4d1ec0d1f 100644 --- a/app/controllers/admin/trends/tags_controller.rb +++ b/app/controllers/admin/trends/tags_controller.rb @@ -5,11 +5,11 @@ class Admin::Trends::TagsController < Admin::BaseController authorize :tag, :index? @tags = filtered_tags.page(params[:page]) - @form = Form::TagBatch.new + @form = Trends::TagBatch.new end def batch - @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) + @form = Trends::TagBatch.new(trends_tag_batch_params.merge(current_account: current_account, action: action_from_button)) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') @@ -20,15 +20,15 @@ class Admin::Trends::TagsController < Admin::BaseController private def filtered_tags - TagFilter.new(filter_params).results + Trends::TagFilter.new(filter_params).results end def filter_params - params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) + params.slice(:page, *Trends::TagFilter::KEYS).permit(:page, *Trends::TagFilter::KEYS) end - def form_tag_batch_params - params.require(:form_tag_batch).permit(:action, tag_ids: []) + def trends_tag_batch_params + params.require(:trends_tag_batch).permit(:action, tag_ids: []) end def action_from_button diff --git a/app/controllers/api/v1/admin/trends/links_controller.rb b/app/controllers/api/v1/admin/trends/links_controller.rb new file mode 100644 index 0000000000..63b3d9358e --- /dev/null +++ b/app/controllers/api/v1/admin/trends/links_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Api::V1::Admin::Trends::LinksController < Api::BaseController + protect_from_forgery with: :exception + + before_action -> { authorize_if_got_token! :'admin:read' } + before_action :require_staff! + before_action :set_links + + def index + render json: @links, each_serializer: REST::Trends::LinkSerializer + end + + private + + def set_links + @links = Trends.links.query.limit(limit_param(10)) + end +end diff --git a/app/controllers/api/v1/admin/trends/statuses_controller.rb b/app/controllers/api/v1/admin/trends/statuses_controller.rb new file mode 100644 index 0000000000..86633cc743 --- /dev/null +++ b/app/controllers/api/v1/admin/trends/statuses_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Api::V1::Admin::Trends::StatusesController < Api::BaseController + protect_from_forgery with: :exception + + before_action -> { authorize_if_got_token! :'admin:read' } + before_action :require_staff! + before_action :set_statuses + + def index + render json: @statuses, each_serializer: REST::StatusSerializer + end + + private + + def set_statuses + @statuses = cache_collection(Trends.statuses.query.limit(limit_param(DEFAULT_STATUSES_LIMIT)), Status) + end +end diff --git a/app/controllers/api/v1/admin/trends/tags_controller.rb b/app/controllers/api/v1/admin/trends/tags_controller.rb index 4815af31ea..5cc4c269d7 100644 --- a/app/controllers/api/v1/admin/trends/tags_controller.rb +++ b/app/controllers/api/v1/admin/trends/tags_controller.rb @@ -14,6 +14,6 @@ class Api::V1::Admin::Trends::TagsController < Api::BaseController private def set_tags - @tags = Trends.tags.get(false, limit_param(10)) + @tags = Trends.tags.query.limit(limit_param(10)) end end diff --git a/app/controllers/api/v1/trends/links_controller.rb b/app/controllers/api/v1/trends/links_controller.rb index 1c3ab1e1c2..ad20e7f8be 100644 --- a/app/controllers/api/v1/trends/links_controller.rb +++ b/app/controllers/api/v1/trends/links_controller.rb @@ -12,10 +12,14 @@ class Api::V1::Trends::LinksController < Api::BaseController def set_links @links = begin if Setting.trends - Trends.links.get(true, limit_param(10)) + links_from_trends else [] end end end + + def links_from_trends + Trends.links.query.allowed.in_locale(content_locale).limit(limit_param(10)) + end end diff --git a/app/controllers/api/v1/trends/statuses_controller.rb b/app/controllers/api/v1/trends/statuses_controller.rb new file mode 100644 index 0000000000..d4ec97ae5f --- /dev/null +++ b/app/controllers/api/v1/trends/statuses_controller.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class Api::V1::Trends::StatusesController < Api::BaseController + before_action :set_statuses + + def index + render json: @statuses, each_serializer: REST::StatusSerializer + end + + private + + def set_statuses + @statuses = begin + if Setting.trends + cache_collection(statuses_from_trends, Status) + else + [] + end + end + end + + def statuses_from_trends + scope = Trends.statuses.query.allowed.in_locale(content_locale) + scope = scope.filtered_for(current_account) if user_signed_in? + scope.limit(limit_param(DEFAULT_STATUSES_LIMIT)) + end +end diff --git a/app/controllers/api/v1/trends/tags_controller.rb b/app/controllers/api/v1/trends/tags_controller.rb index 947b53de23..1334b72d25 100644 --- a/app/controllers/api/v1/trends/tags_controller.rb +++ b/app/controllers/api/v1/trends/tags_controller.rb @@ -12,7 +12,7 @@ class Api::V1::Trends::TagsController < Api::BaseController def set_tags @tags = begin if Setting.trends - Trends.tags.get(true, limit_param(10)) + Trends.tags.query.allowed.limit(limit_param(10)) else [] end diff --git a/app/controllers/concerns/localized.rb b/app/controllers/concerns/localized.rb index 173316800f..ede299d5a4 100644 --- a/app/controllers/concerns/localized.rb +++ b/app/controllers/concerns/localized.rb @@ -27,4 +27,8 @@ module Localized def available_locale_or_nil(locale_name) locale_name.to_sym if locale_name.present? && I18n.available_locales.include?(locale_name.to_sym) end + + def content_locale + @content_locale ||= I18n.locale.to_s.split(/[_-]/).first + end end diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb index 907529b372..140fc73ede 100644 --- a/app/helpers/admin/filter_helper.rb +++ b/app/helpers/admin/filter_helper.rb @@ -5,9 +5,10 @@ module Admin::FilterHelper AccountFilter::KEYS, CustomEmojiFilter::KEYS, ReportFilter::KEYS, - TagFilter::KEYS, - PreviewCardProviderFilter::KEYS, - PreviewCardFilter::KEYS, + Trends::TagFilter::KEYS, + Trends::PreviewCardProviderFilter::KEYS, + Trends::PreviewCardFilter::KEYS, + Trends::StatusFilter::KEYS, InstanceFilter::KEYS, InviteFilter::KEYS, RelationshipFilter::KEYS, diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb index 3a65af6869..f22cc6d284 100644 --- a/app/helpers/languages_helper.rb +++ b/app/helpers/languages_helper.rb @@ -242,6 +242,6 @@ module LanguagesHelper end def valid_locale?(locale) - SUPPORTED_LOCALES.key?(locale.to_sym) + locale.present? && SUPPORTED_LOCALES.key?(locale.to_sym) end end diff --git a/app/javascript/styles/mastodon/accounts.scss b/app/javascript/styles/mastodon/accounts.scss index 485fe4a9dc..215774a192 100644 --- a/app/javascript/styles/mastodon/accounts.scss +++ b/app/javascript/styles/mastodon/accounts.scss @@ -331,7 +331,8 @@ } .batch-table__row--muted .pending-account__header, -.batch-table__row--muted .accounts-table { +.batch-table__row--muted .accounts-table, +.batch-table__row--muted .name-tag { &, a, strong { @@ -339,6 +340,10 @@ } } +.batch-table__row--muted .name-tag .avatar { + opacity: 0.5; +} + .batch-table__row--muted .accounts-table { tbody td.accounts-table__extra, &__count, @@ -352,7 +357,8 @@ } .batch-table__row--attention .pending-account__header, -.batch-table__row--attention .accounts-table { +.batch-table__row--attention .accounts-table, +.batch-table__row--attention .name-tag { &, a, strong { diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss index 36bc07a72b..1f7e71776e 100644 --- a/app/javascript/styles/mastodon/tables.scss +++ b/app/javascript/styles/mastodon/tables.scss @@ -210,6 +210,7 @@ a.table-action-link { &__content { padding-top: 12px; padding-bottom: 16px; + overflow: hidden; &--unpadded { padding: 0; @@ -296,3 +297,9 @@ a.table-action-link { } } } + +.one-liner { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 12fad8da40..7cd5a41e8e 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -23,8 +23,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity visibility: visibility_from_audience ) - Trends.tags.register(@status) - Trends.links.register(@status) + Trends.register!(@status) distribute end diff --git a/app/lib/activitypub/activity/like.rb b/app/lib/activitypub/activity/like.rb index c065f01f86..ebbda15b9f 100644 --- a/app/lib/activitypub/activity/like.rb +++ b/app/lib/activitypub/activity/like.rb @@ -7,6 +7,8 @@ class ActivityPub::Activity::Like < ActivityPub::Activity return if original_status.nil? || !original_status.account.local? || delete_arrived_first?(@json['id']) || @account.favourited?(original_status) favourite = original_status.favourites.create!(account: @account) + NotifyService.new.call(original_status.account, :favourite, favourite) + Trends.statuses.register(original_status) end end diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index a9d00c000d..f416977d85 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -35,25 +35,18 @@ class AdminMailer < ApplicationMailer end end - def new_trending_tags(recipient, tags) - @tags = tags - @me = recipient - @instance = Rails.configuration.x.local_domain - @lowest_trending_tag = Trends.tags.get(true, Trends.tags.options[:review_threshold]).last + def new_trends(recipient, links, tags, statuses) + @links = links + @lowest_trending_link = Trends.links.query.allowed.limit(Trends.links.options[:review_threshold]).last + @tags = tags + @lowest_trending_tag = Trends.tags.query.allowed.limit(Trends.tags.options[:review_threshold]).last + @statuses = statuses + @lowest_trending_status = Trends.statuses.query.allowed.limit(Trends.statuses.options[:review_threshold]).last + @me = recipient + @instance = Rails.configuration.x.local_domain locale_for_account(@me) do - mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance) - end - end - - def new_trending_links(recipient, links) - @links = links - @me = recipient - @instance = Rails.configuration.x.local_domain - @lowest_trending_link = Trends.links.get(true, Trends.links.options[:review_threshold]).last - - locale_for_account(@me) do - mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance) + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance) end end end diff --git a/app/models/account.rb b/app/models/account.rb index 2ad45feda0..dfdf9045ff 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -40,13 +40,15 @@ # also_known_as :string is an Array # silenced_at :datetime # suspended_at :datetime -# trust_level :integer # hide_collections :boolean # avatar_storage_schema_version :integer # header_storage_schema_version :integer # devices_url :string # suspension_origin :integer # sensitized_at :datetime +# trendable :boolean +# reviewed_at :datetime +# requested_review_at :datetime # class Account < ApplicationRecord @@ -56,6 +58,7 @@ class Account < ApplicationRecord remote_url salmon_url hub_url + trust_level ) USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i @@ -74,11 +77,6 @@ class Account < ApplicationRecord include DomainMaterializable include AccountMerging - TRUST_LEVELS = { - untrusted: 0, - trusted: 1, - }.freeze - enum protocol: [:ostatus, :activitypub] enum suspension_origin: [:local, :remote], _prefix: true @@ -202,10 +200,6 @@ class Account < ApplicationRecord last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago end - def trust_level - self[:trust_level] || 0 - end - def refresh! ResolveAccountService.new.call(acct) unless local? end @@ -388,6 +382,22 @@ class Account < ApplicationRecord @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/" end + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def requires_review_notification? + requires_review? && !requested_review? + end + class Field < ActiveModelSerializers::Model attributes :name, :value, :verified_at, :account diff --git a/app/models/status.rb b/app/models/status.rb index 96e41b1d32..adb92ef916 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -268,6 +268,18 @@ class Status < ApplicationRecord update_status_stat!(key => [public_send(key) - 1, 0].max) end + def trendable? + if attributes['trendable'].nil? + account.trendable? + else + attributes['trendable'] + end + end + + def requires_review_notification? + attributes['trendable'].nil? && account.requires_review_notification? + end + after_create_commit :increment_counter_caches after_destroy_commit :decrement_counter_caches diff --git a/app/models/trends.rb b/app/models/trends.rb index 7dd3a9c87e..f8864e55f5 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -13,15 +13,37 @@ module Trends @tags ||= Trends::Tags.new end + def self.statuses + @statuses ||= Trends::Statuses.new + end + + def self.register!(status) + [links, tags, statuses].each { |trend_type| trend_type.register(status) } + end + def self.refresh! - [links, tags].each(&:refresh) + [links, tags, statuses].each(&:refresh) end def self.request_review! - [links, tags].each(&:request_review) if enabled? + return unless enabled? + + links_requiring_review = links.request_review + tags_requiring_review = tags.request_review + statuses_requiring_review = statuses.request_review + + return if links_requiring_review.empty? && tags_requiring_review.empty? && statuses_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trends(user.account, links_requiring_review, tags_requiring_review, statuses_requiring_review).deliver_later! if user.allows_trends_review_emails? + end end def self.enabled? Setting.trends end + + def self.available_locales + @available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq + end end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index b767dcb1ac..7ed13228df 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -2,6 +2,7 @@ class Trends::Base include Redisable + include LanguagesHelper class_attribute :default_options @@ -32,8 +33,8 @@ class Trends::Base raise NotImplementedError end - def get(*) - raise NotImplementedError + def query + Trends::Query.new(key_prefix, klass) end def score(id) @@ -72,6 +73,21 @@ class Trends::Base redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 end + # @param [Integer] id + # @param [Float] score + # @param [Hash] subsets + def add_to_and_remove_from_subsets(id, score, subsets = {}) + subsets.each_key do |subset| + key = [key_prefix, subset].compact.join(':') + + if score.positive? && subsets[subset] + redis.zadd(key, score, id) + else + redis.zrem(key, id) + end + end + end + private def used_key(at_time) diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index a0d65138ba..62308e7067 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -4,8 +4,8 @@ class Trends::Links < Trends::Base PREFIX = 'trending_links' self.default_options = { - threshold: 15, - review_threshold: 10, + threshold: 5, + review_threshold: 3, max_score_cooldown: 2.days.freeze, max_score_halflife: 8.hours.freeze, } @@ -27,12 +27,6 @@ class Trends::Links < Trends::Base record_used_id(preview_card.id, at_time) end - def get(allowed, limit) - preview_card_ids = currently_trending_ids(allowed, limit) - preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) - preview_card_ids.map { |id| preview_cards[id] }.compact - end - def refresh(at_time = Time.now.utc) preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) calculate_scores(preview_cards, at_time) @@ -42,7 +36,7 @@ class Trends::Links < Trends::Base def request_review preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) - preview_cards_requiring_review = preview_cards.filter_map do |preview_card| + preview_cards.filter_map do |preview_card| next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? if preview_card.provider.nil? @@ -53,12 +47,6 @@ class Trends::Links < Trends::Base preview_card end - - return if preview_cards_requiring_review.empty? - - User.staff.includes(:account).find_each do |user| - AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? - end end protected @@ -67,6 +55,10 @@ class Trends::Links < Trends::Base PREFIX end + def klass + PreviewCard + end + private def calculate_scores(preview_cards, at_time) @@ -96,17 +88,27 @@ class Trends::Links < Trends::Base decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - if decaying_score.zero? - redis.zrem("#{PREFIX}:all", preview_card.id) - redis.zrem("#{PREFIX}:allowed", preview_card.id) - else - redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) + add_to_and_remove_from_subsets(preview_card.id, decaying_score, { + all: true, + allowed: preview_card.trendable?, + }) - if preview_card.trendable? - redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) - else - redis.zrem("#{PREFIX}:allowed", preview_card.id) - end + next unless valid_locale?(preview_card.language) + + add_to_and_remove_from_subsets(preview_card.id, decaying_score, { + "all:#{preview_card.language}" => true, + "allowed:#{preview_card.language}" => preview_card.trendable?, + }) + end + + # Clean up localized sets by calculating the intersection with the main + # set. We do this instead of just deleting the localized sets to avoid + # having moments where the API returns empty results + + redis.pipelined do + Trends.available_locales.each do |locale| + redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') + redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max') end end end diff --git a/app/models/form/preview_card_batch.rb b/app/models/trends/preview_card_batch.rb similarity index 82% rename from app/models/form/preview_card_batch.rb rename to app/models/trends/preview_card_batch.rb index 5f6e6522a2..b1d6829101 100644 --- a/app/models/form/preview_card_batch.rb +++ b/app/models/trends/preview_card_batch.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Form::PreviewCardBatch +class Trends::PreviewCardBatch include ActiveModel::Model include Authorization @@ -10,12 +10,12 @@ class Form::PreviewCardBatch case action when 'approve' approve! - when 'approve_all' - approve_all! + when 'approve_providers' + approve_providers! when 'reject' reject! - when 'reject_all' - reject_all! + when 'reject_providers' + reject_providers! end end @@ -30,13 +30,13 @@ class Form::PreviewCardBatch end def approve! - preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.each { |preview_card| authorize(preview_card, :review?) } preview_cards.update_all(trendable: true) end - def approve_all! + def approve_providers! preview_card_providers.each do |provider| - authorize(provider, :update?) + authorize(provider, :review?) provider.update(trendable: true, reviewed_at: action_time) end @@ -45,13 +45,13 @@ class Form::PreviewCardBatch end def reject! - preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.each { |preview_card| authorize(preview_card, :review?) } preview_cards.update_all(trendable: false) end - def reject_all! + def reject_providers! preview_card_providers.each do |provider| - authorize(provider, :update?) + authorize(provider, :review?) provider.update(trendable: false, reviewed_at: action_time) end diff --git a/app/models/preview_card_filter.rb b/app/models/trends/preview_card_filter.rb similarity index 52% rename from app/models/preview_card_filter.rb rename to app/models/trends/preview_card_filter.rb index 8dda9989ce..25add58c89 100644 --- a/app/models/preview_card_filter.rb +++ b/app/models/trends/preview_card_filter.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true -class PreviewCardFilter +class Trends::PreviewCardFilter KEYS = %i( trending + locale ).freeze attr_reader :params @@ -15,7 +16,7 @@ class PreviewCardFilter scope = PreviewCard.unscoped params.each do |key, value| - next if key.to_s == 'page' + next if %w(page locale).include?(key.to_s) scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end @@ -35,19 +36,11 @@ class PreviewCardFilter end def trending_scope(value) - ids = begin - case value.to_s - when 'allowed' - Trends.links.currently_trending_ids(true, -1) - else - Trends.links.currently_trending_ids(false, -1) - end - end + scope = Trends.links.query - if ids.empty? - PreviewCard.none - else - PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') - end + scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? + scope = scope.allowed if value == 'allowed' + + scope.to_arel end end diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/trends/preview_card_provider_batch.rb similarity index 91% rename from app/models/form/preview_card_provider_batch.rb rename to app/models/trends/preview_card_provider_batch.rb index e6ab3d8fa3..062720c814 100644 --- a/app/models/form/preview_card_provider_batch.rb +++ b/app/models/trends/preview_card_provider_batch.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Form::PreviewCardProviderBatch +class Trends::PreviewCardProviderBatch include ActiveModel::Model include Authorization @@ -22,12 +22,12 @@ class Form::PreviewCardProviderBatch end def approve! - preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.each { |provider| authorize(provider, :review?) } preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) end def reject! - preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.each { |provider| authorize(provider, :review?) } preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) end end diff --git a/app/models/preview_card_provider_filter.rb b/app/models/trends/preview_card_provider_filter.rb similarity index 95% rename from app/models/preview_card_provider_filter.rb rename to app/models/trends/preview_card_provider_filter.rb index 1e90d3c9da..abfdd07e88 100644 --- a/app/models/preview_card_provider_filter.rb +++ b/app/models/trends/preview_card_provider_filter.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PreviewCardProviderFilter +class Trends::PreviewCardProviderFilter KEYS = %i( status ).freeze diff --git a/app/models/trends/query.rb b/app/models/trends/query.rb new file mode 100644 index 0000000000..64a4c0c1fb --- /dev/null +++ b/app/models/trends/query.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +class Trends::Query + include Redisable + include Enumerable + + attr_reader :prefix, :klass, :loaded + + alias loaded? loaded + + def initialize(prefix, klass) + @prefix = prefix + @klass = klass + @records = [] + @loaded = false + @allowed = false + @limit = -1 + @offset = 0 + end + + def allowed! + @allowed = true + self + end + + def allowed + clone.allowed! + end + + def in_locale!(value) + @locale = value + self + end + + def in_locale(value) + clone.in_locale!(value) + end + + def offset!(value) + @offset = value + self + end + + def offset(value) + clone.offset!(value) + end + + def limit!(value) + @limit = value + self + end + + def limit(value) + clone.limit!(value) + end + + def records + load + @records + end + + delegate :each, :empty?, :first, :last, to: :records + + def to_ary + records.dup + end + + alias to_a to_ary + + def to_arel + tmp_ids = ids + + if tmp_ids.empty? + klass.none + else + klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering') + end + end + + private + + def key + [@prefix, @allowed ? 'allowed' : 'all', @locale].compact.join(':') + end + + def load + unless loaded? + @records = perform_queries + @loaded = true + end + + self + end + + def ids + redis.zrevrange(key, @offset, @limit.positive? ? @limit - 1 : @limit).map(&:to_i) + end + + def perform_queries + apply_scopes(to_arel).to_a + end + + def apply_scopes(scope) + scope + end +end diff --git a/app/models/trends/status_batch.rb b/app/models/trends/status_batch.rb new file mode 100644 index 0000000000..78d93bed44 --- /dev/null +++ b/app/models/trends/status_batch.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Trends::StatusBatch + include ActiveModel::Model + include Authorization + + attr_accessor :status_ids, :action, :current_account + + def save + case action + when 'approve' + approve! + when 'approve_accounts' + approve_accounts! + when 'reject' + reject! + when 'reject_accounts' + reject_accounts! + end + end + + private + + def statuses + @statuses ||= Status.where(id: status_ids) + end + + def status_accounts + @status_accounts ||= Account.where(id: statuses.map(&:account_id).uniq) + end + + def approve! + statuses.each { |status| authorize(status, :review?) } + statuses.update_all(trendable: true) + end + + def approve_accounts! + status_accounts.each do |account| + authorize(account, :review?) + account.update(trendable: true, reviewed_at: action_time) + end + + # Reset any individual overrides + statuses.update_all(trendable: nil) + end + + def reject! + statuses.each { |status| authorize(status, :review?) } + statuses.update_all(trendable: false) + end + + def reject_accounts! + status_accounts.each do |account| + authorize(account, :review?) + account.update(trendable: false, reviewed_at: action_time) + end + + # Reset any individual overrides + statuses.update_all(trendable: nil) + end + + def action_time + @action_time ||= Time.now.utc + end +end diff --git a/app/models/trends/status_filter.rb b/app/models/trends/status_filter.rb new file mode 100644 index 0000000000..7c453e3393 --- /dev/null +++ b/app/models/trends/status_filter.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class Trends::StatusFilter + KEYS = %i( + trending + locale + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = Status.unscoped.kept + + params.each do |key, value| + next if %w(page locale).include?(key.to_s) + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'trending' + trending_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def trending_scope(value) + scope = Trends.statuses.query + + scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? + scope = scope.allowed if value == 'allowed' + + scope.to_arel + end +end diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb new file mode 100644 index 0000000000..e785413ec4 --- /dev/null +++ b/app/models/trends/statuses.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +class Trends::Statuses < Trends::Base + PREFIX = 'trending_statuses' + + self.default_options = { + threshold: 5, + review_threshold: 3, + score_halflife: 2.hours.freeze, + } + + class Query < Trends::Query + def filtered_for!(account) + @account = account + self + end + + def filtered_for(account) + clone.filtered_for!(account) + end + + private + + def apply_scopes(scope) + scope.includes(:account) + end + + def perform_queries + return super if @account.nil? + + statuses = super + account_ids = statuses.map(&:account_id) + account_domains = statuses.map(&:account_domain) + + preloaded_relations = { + blocking: Account.blocking_map(account_ids, @account.id), + blocked_by: Account.blocked_by_map(account_ids, @account.id), + muting: Account.muting_map(account_ids, @account.id), + following: Account.following_map(account_ids, @account.id), + domain_blocking_by_domain: Account.domain_blocking_map_by_domain(account_domains, @account.id), + } + + statuses.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? } + end + end + + def register(status, at_time = Time.now.utc) + add(status.proper, status.account_id, at_time) if eligible?(status) + end + + def add(status, _account_id, at_time = Time.now.utc) + # We rely on the total reblogs and favourites count, so we + # don't record which account did the what and when here + + record_used_id(status.id, at_time) + end + + def query + Query.new(key_prefix, klass) + end + + def refresh(at_time = Time.now.utc) + statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments) + calculate_scores(statuses, at_time) + trim_older_items + end + + def request_review + statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account) + + statuses.filter_map do |status| + next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification? + + status.account.touch(:requested_review_at) + status + end + end + + protected + + def key_prefix + PREFIX + end + + def klass + Status + end + + private + + def eligible?(status) + original_status = status.proper + + original_status.public_visibility? && + original_status.account.discoverable? && !original_status.account.silenced? && + original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply? + end + + def calculate_scores(statuses, at_time) + redis.pipelined do + statuses.each do |status| + expected = 1.0 + observed = (status.reblogs_count + status.favourites_count).to_f + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) + + add_to_and_remove_from_subsets(status.id, decaying_score, { + all: true, + allowed: status.trendable? && status.account.discoverable?, + }) + + next unless valid_locale?(status.language) + + add_to_and_remove_from_subsets(status.id, decaying_score, { + "all:#{status.language}" => true, + "allowed:#{status.language}" => status.trendable? && status.account.discoverable?, + }) + end + + # Clean up localized sets by calculating the intersection with the main + # set. We do this instead of just deleting the localized sets to avoid + # having moments where the API returns empty results + + Trends.available_locales.each do |locale| + redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') + redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max') + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/form/tag_batch.rb b/app/models/trends/tag_batch.rb similarity index 81% rename from app/models/form/tag_batch.rb rename to app/models/trends/tag_batch.rb index b9330745fe..16ee08c065 100644 --- a/app/models/form/tag_batch.rb +++ b/app/models/trends/tag_batch.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Form::TagBatch +class Trends::TagBatch include ActiveModel::Model include Authorization @@ -22,12 +22,12 @@ class Form::TagBatch end def approve! - tags.each { |tag| authorize(tag, :update?) } + tags.each { |tag| authorize(tag, :review?) } tags.update_all(trendable: true, reviewed_at: action_time) end def reject! - tags.each { |tag| authorize(tag, :update?) } + tags.each { |tag| authorize(tag, :review?) } tags.update_all(trendable: false, reviewed_at: action_time) end diff --git a/app/models/tag_filter.rb b/app/models/trends/tag_filter.rb similarity index 76% rename from app/models/tag_filter.rb rename to app/models/trends/tag_filter.rb index ecdb52503e..3b142efc45 100644 --- a/app/models/tag_filter.rb +++ b/app/models/trends/tag_filter.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class TagFilter +class Trends::TagFilter KEYS = %i( trending status @@ -42,13 +42,7 @@ class TagFilter end def trending_scope - ids = Trends.tags.currently_trending_ids(false, -1) - - if ids.empty? - Tag.none - else - Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering') - end + Trends.tags.query.to_arel end def status_scope(value) diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb index 2ea4550df6..3caa58815d 100644 --- a/app/models/trends/tags.rb +++ b/app/models/trends/tags.rb @@ -5,7 +5,7 @@ class Trends::Tags < Trends::Base self.default_options = { threshold: 5, - review_threshold: 10, + review_threshold: 3, max_score_cooldown: 2.days.freeze, max_score_halflife: 4.hours.freeze, } @@ -29,27 +29,15 @@ class Trends::Tags < Trends::Base trim_older_items end - def get(allowed, limit) - tag_ids = currently_trending_ids(allowed, limit) - tags = Tag.where(id: tag_ids).index_by(&:id) - tag_ids.map { |id| tags[id] }.compact - end - def request_review tags = Tag.where(id: currently_trending_ids(false, -1)) - tags_requiring_review = tags.filter_map do |tag| + tags.filter_map do |tag| next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? tag.touch(:requested_review_at) tag end - - return if tags_requiring_review.empty? - - User.staff.includes(:account).find_each do |user| - AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? - end end protected @@ -58,6 +46,10 @@ class Trends::Tags < Trends::Base PREFIX end + def klass + Tag + end + private def calculate_scores(tags, at_time) @@ -87,18 +79,10 @@ class Trends::Tags < Trends::Base decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - if decaying_score.zero? - redis.zrem("#{PREFIX}:all", tag.id) - redis.zrem("#{PREFIX}:allowed", tag.id) - else - redis.zadd("#{PREFIX}:all", decaying_score, tag.id) - - if tag.trendable? - redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) - else - redis.zrem("#{PREFIX}:allowed", tag.id) - end - end + add_to_and_remove_from_subsets(tag.id, decaying_score, { + all: true, + allowed: tag.trendable?, + }) end end diff --git a/app/models/user.rb b/app/models/user.rb index 517254a910..bbf850d84d 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -269,7 +269,7 @@ class User < ApplicationRecord settings.notification_emails['appeal'] end - def allows_trending_tag_emails? + def allows_trends_review_emails? settings.notification_emails['trending_tag'] end diff --git a/app/policies/account_policy.rb b/app/policies/account_policy.rb index 46237e45c1..cc23771e7a 100644 --- a/app/policies/account_policy.rb +++ b/app/policies/account_policy.rb @@ -68,4 +68,8 @@ class AccountPolicy < ApplicationPolicy def unblock_email? staff? end + + def review? + staff? + end end diff --git a/app/policies/preview_card_policy.rb b/app/policies/preview_card_policy.rb index 4f485d7fc6..0410987e46 100644 --- a/app/policies/preview_card_policy.rb +++ b/app/policies/preview_card_policy.rb @@ -5,7 +5,7 @@ class PreviewCardPolicy < ApplicationPolicy staff? end - def update? + def review? staff? end end diff --git a/app/policies/preview_card_provider_policy.rb b/app/policies/preview_card_provider_policy.rb index 598d54a5e2..44d2ad5cfc 100644 --- a/app/policies/preview_card_provider_policy.rb +++ b/app/policies/preview_card_provider_policy.rb @@ -5,7 +5,7 @@ class PreviewCardProviderPolicy < ApplicationPolicy staff? end - def update? + def review? staff? end end diff --git a/app/policies/status_policy.rb b/app/policies/status_policy.rb index 6e9b840db6..400f1ec791 100644 --- a/app/policies/status_policy.rb +++ b/app/policies/status_policy.rb @@ -41,6 +41,10 @@ class StatusPolicy < ApplicationPolicy staff? || owned? end + def review? + staff? + end + private def requires_mention? diff --git a/app/policies/tag_policy.rb b/app/policies/tag_policy.rb index aaf70fcabc..bdfcec0c96 100644 --- a/app/policies/tag_policy.rb +++ b/app/policies/tag_policy.rb @@ -12,4 +12,8 @@ class TagPolicy < ApplicationPolicy def update? staff? end + + def review? + staff? + end end diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index a572a7c597..a2d535d262 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -220,21 +220,23 @@ class DeleteAccountService < BaseService return unless keep_account_record? - @account.silenced_at = nil - @account.suspended_at = @options[:suspended_at] || Time.now.utc - @account.suspension_origin = :local - @account.locked = false - @account.memorial = false - @account.discoverable = false - @account.display_name = '' - @account.note = '' - @account.fields = [] - @account.statuses_count = 0 - @account.followers_count = 0 - @account.following_count = 0 - @account.moved_to_account = nil - @account.also_known_as = [] - @account.trust_level = :untrusted + @account.silenced_at = nil + @account.suspended_at = @options[:suspended_at] || Time.now.utc + @account.suspension_origin = :local + @account.locked = false + @account.memorial = false + @account.discoverable = false + @account.trendable = false + @account.display_name = '' + @account.note = '' + @account.fields = [] + @account.statuses_count = 0 + @account.followers_count = 0 + @account.following_count = 0 + @account.moved_to_account = nil + @account.reviewed_at = nil + @account.requested_review_at = nil + @account.also_known_as = [] @account.avatar.destroy @account.header.destroy @account.save! diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb index a0ab3b4b75..0ca0081b48 100644 --- a/app/services/favourite_service.rb +++ b/app/services/favourite_service.rb @@ -17,6 +17,8 @@ class FavouriteService < BaseService favourite = Favourite.create!(account: account, status: status) + Trends.statuses.register(status) + create_notification(favourite) bump_potential_friendship(account, status) diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 2d1265f107..7d2981709b 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -30,8 +30,7 @@ class ReblogService < BaseService reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit]) - Trends.tags.register(reblog) - Trends.links.register(reblog) + Trends.register!(reblog) DistributionWorker.perform_async(reblog.id) ActivityPub::DistributionWorker.perform_async(reblog.id) diff --git a/app/views/admin/custom_emojis/_custom_emoji.html.haml b/app/views/admin/custom_emojis/_custom_emoji.html.haml index 526c844e92..41f3975cf4 100644 --- a/app/views/admin/custom_emojis/_custom_emoji.html.haml +++ b/app/views/admin/custom_emojis/_custom_emoji.html.haml @@ -3,7 +3,7 @@ = f.check_box :custom_emoji_ids, { multiple: true, include_hidden: false }, custom_emoji.id .batch-table__row__content.batch-table__row__content--with-image .batch-table__row__content__image - = custom_emoji_tag(custom_emoji, animate = current_account&.user&.setting_auto_play_gif) + = custom_emoji_tag(custom_emoji, current_account&.user&.setting_auto_play_gif) .batch-table__row__content__text %samp= ":#{custom_emoji.shortcode}:" diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml index 272681864a..ebc4a2c6b8 100644 --- a/app/views/admin/follow_recommendations/show.html.haml +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -9,12 +9,14 @@ %hr.spacer/ = form_tag admin_follow_recommendations_path, method: 'GET', class: 'simple_form' do + - RelationshipFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + .filters .filter-subset.filter-subset--with-select %strong= t('admin.follow_recommendations.language') .input.select.optional - = select_tag :language, options_for_select(I18n.available_locales.map { |key| key.to_s.split(/[_-]/).first.to_sym }.uniq.map { |key| [standard_locale_name(key), key]}, @language) - + = select_tag :language, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, @language) .filter-subset %strong= t('admin.follow_recommendations.status') %ul diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml index 240ae722b5..79f3513d31 100644 --- a/app/views/admin/trends/links/index.html.haml +++ b/app/views/admin/trends/links/index.html.haml @@ -4,23 +4,29 @@ - content_for :header_tags do = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' -.filters - .filter-subset - %strong= t('admin.trends.trending') - %ul - %li= filter_link_to t('generic.all'), trending: nil - %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' - .back-link - = link_to admin_trends_links_preview_card_providers_path do - = t('admin.trends.preview_card_providers.title') - = fa_icon 'chevron-right fw' += form_tag admin_trends_links_path, method: 'GET', class: 'simple_form' do + - Trends::PreviewCardFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? -%hr.spacer/ + .filters + .filter-subset.filter-subset--with-select + %strong= t('admin.follow_recommendations.language') + .input.select.optional + = select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, params[:locale]), include_blank: true + .filter-subset + %strong= t('admin.trends.trending') + %ul + %li= filter_link_to t('generic.all'), trending: nil + %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' + .back-link + = link_to admin_trends_links_preview_card_providers_path do + = t('admin.trends.preview_card_providers.title') + = fa_icon 'chevron-right fw' = form_for(@form, url: batch_admin_trends_links_path) do |f| = hidden_field_tag :page, params[:page] || 1 - - PreviewCardFilter::KEYS.each do |key| + - Trends::PreviewCardFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? .batch-table @@ -29,9 +35,9 @@ = check_box_tag :batch_checkbox_all, nil, false .batch-table__toolbar__actions = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_providers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_providers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } .batch-table__body - if @preview_cards.empty? = nothing_here 'nothing-here--under-tabs' diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml index eac6e641fe..b793499477 100644 --- a/app/views/admin/trends/links/preview_card_providers/index.html.haml +++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml @@ -23,7 +23,7 @@ = form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f| = hidden_field_tag :page, params[:page] || 1 - - PreviewCardProviderFilter::KEYS.each do |key| + - Trends::PreviewCardProviderFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? .batch-table.optional diff --git a/app/views/admin/trends/statuses/_status.html.haml b/app/views/admin/trends/statuses/_status.html.haml new file mode 100644 index 0000000000..c99ee5d60e --- /dev/null +++ b/app/views/admin/trends/statuses/_status.html.haml @@ -0,0 +1,30 @@ +.batch-table__row{ class: [status.account.requires_review? && 'batch-table__row--attention', !status.account.requires_review? && !status.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :status_ids, { multiple: true, include_hidden: false }, status.id + + .batch-table__row__content.pending-account__header + .one-liner + = admin_account_link_to status.account + + = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank', class: 'emojify', rel: 'noopener noreferrer' do + = one_line_preview(status) + + - status.media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name + + = t('admin.trends.statuses.shared_by', count: status.reblogs_count + status.favourites_count, friendly_count: friendly_number_to_human(status.reblogs_count + status.favourites_count)) + + - if status.account.domain.present? + • + = status.account.domain + - if status.language.present? + • + = standard_locale_name(status.language) + - if status.trendable? && (rank = Trends.statuses.rank(status.id)) + • + %abbr{ title: t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) + - elsif status.account.requires_review? + • + = t('admin.trends.pending_review') diff --git a/app/views/admin/trends/statuses/index.html.haml b/app/views/admin/trends/statuses/index.html.haml new file mode 100644 index 0000000000..3476882628 --- /dev/null +++ b/app/views/admin/trends/statuses/index.html.haml @@ -0,0 +1,43 @@ +- content_for :page_title do + = t('admin.trends.statuses.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + += form_tag admin_trends_statuses_path, method: 'GET', class: 'simple_form' do + - Trends::StatusFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .filters + .filter-subset.filter-subset--with-select + %strong= t('admin.follow_recommendations.language') + .input.select.optional + = select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key]}, params[:locale]), include_blank: true + .filter-subset + %strong= t('admin.trends.trending') + %ul + %li= filter_link_to t('generic.all'), trending: nil + %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' + += form_for(@form, url: batch_admin_trends_statuses_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - Trends::StatusFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.statuses.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('check'), t('admin.trends.statuses.allow_account')]), name: :approve_accounts, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.statuses.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.statuses.disallow_account')]), name: :reject_accounts, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @statuses.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'status', collection: @statuses, locals: { f: f } + += paginate @statuses diff --git a/app/views/admin/trends/tags/index.html.haml b/app/views/admin/trends/tags/index.html.haml index 8df0a9920b..8a2f785bc2 100644 --- a/app/views/admin/trends/tags/index.html.haml +++ b/app/views/admin/trends/tags/index.html.haml @@ -13,12 +13,10 @@ %li= filter_link_to t('admin.trends.rejected'), status: 'rejected' %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review' -%hr.spacer/ - = form_for(@form, url: batch_admin_trends_tags_path) do |f| = hidden_field_tag :page, params[:page] || 1 - - TagFilter::KEYS.each do |key| + - Trends::TagFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? .batch-table.optional diff --git a/app/views/admin_mailer/_new_trending_links.text.erb b/app/views/admin_mailer/_new_trending_links.text.erb new file mode 100644 index 0000000000..405926fddc --- /dev/null +++ b/app/views/admin_mailer/_new_trending_links.text.erb @@ -0,0 +1,14 @@ +<%= raw t('admin_mailer.new_trends.new_trending_links.title') %> + +<% @links.each do |link| %> +- <%= link.title %> • <%= link.url %> + <%= raw t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> +<% end %> + +<% if @lowest_trending_link %> +<%= raw t('admin_mailer.new_trends.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2), rank: Trends.links.options[:review_threshold]) %> +<% else %> +<%= raw t('admin_mailer.new_trends.new_trending_links.no_approved_links') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> diff --git a/app/views/admin_mailer/_new_trending_statuses.text.erb b/app/views/admin_mailer/_new_trending_statuses.text.erb new file mode 100644 index 0000000000..8d11a80c2e --- /dev/null +++ b/app/views/admin_mailer/_new_trending_statuses.text.erb @@ -0,0 +1,14 @@ +<%= raw t('admin_mailer.new_trends.new_trending_statuses.title') %> + +<% @statuses.each do |status| %> +- <%= ActivityPub::TagManager.instance.url_for(status) %> + <%= raw t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id).round(2)) %> +<% end %> + +<% if @lowest_trending_status %> +<%= raw t('admin_mailer.new_trends.new_trending_statuses.requirements', lowest_status_url: ActivityPub::TagManager.instance.url_for(@lowest_trending_status), lowest_status_score: Trends.statuses.score(@lowest_trending_status.id).round(2), rank: Trends.statuses.options[:review_threshold]) %> +<% else %> +<%= raw t('admin_mailer.new_trends.new_trending_statuses.no_approved_statuses') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_statuses_url %> diff --git a/app/views/admin_mailer/_new_trending_tags.text.erb b/app/views/admin_mailer/_new_trending_tags.text.erb new file mode 100644 index 0000000000..49fe843096 --- /dev/null +++ b/app/views/admin_mailer/_new_trending_tags.text.erb @@ -0,0 +1,14 @@ +<%= raw t('admin_mailer.new_trends.new_trending_tags.title') %> + +<% @tags.each do |tag| %> +- #<%= tag.name %> + <%= raw t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> +<% end %> + +<% if @lowest_trending_tag %> +<%= raw t('admin_mailer.new_trends.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2), rank: Trends.tags.options[:review_threshold]) %> +<% else %> +<%= raw t('admin_mailer.new_trends.new_trending_tags.no_approved_tags') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %> diff --git a/app/views/admin_mailer/new_trending_links.text.erb b/app/views/admin_mailer/new_trending_links.text.erb deleted file mode 100644 index 51789aca5c..0000000000 --- a/app/views/admin_mailer/new_trending_links.text.erb +++ /dev/null @@ -1,16 +0,0 @@ -<%= raw t('application_mailer.salutation', name: display_name(@me)) %> - -<%= raw t('admin_mailer.new_trending_links.body') %> - -<% @links.each do |link| %> -- <%= link.title %> • <%= link.url %> - <%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> -<% end %> - -<% if @lowest_trending_link %> -<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %> -<% else %> -<%= t('admin_mailer.new_trending_links.no_approved_links') %> -<% end %> - -<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> diff --git a/app/views/admin_mailer/new_trending_tags.text.erb b/app/views/admin_mailer/new_trending_tags.text.erb deleted file mode 100644 index 9ea31fa7ce..0000000000 --- a/app/views/admin_mailer/new_trending_tags.text.erb +++ /dev/null @@ -1,16 +0,0 @@ -<%= raw t('application_mailer.salutation', name: display_name(@me)) %> - -<%= raw t('admin_mailer.new_trending_tags.body') %> - -<% @tags.each do |tag| %> -- #<%= tag.name %> - <%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> -<% end %> - -<% if @lowest_trending_tag %> -<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %> -<% else %> -<%= t('admin_mailer.new_trending_tags.no_approved_tags') %> -<% end %> - -<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(status: 'pending_review') %> diff --git a/app/views/admin_mailer/new_trends.text.erb b/app/views/admin_mailer/new_trends.text.erb new file mode 100644 index 0000000000..13b2968461 --- /dev/null +++ b/app/views/admin_mailer/new_trends.text.erb @@ -0,0 +1,13 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trends.body') %> + +<% unless @links.empty? %> +<%= render 'new_trending_links' %> +<% end %> +<% unless @tags.empty? %> +<%= render 'new_trending_tags' unless @tags.empty? %> +<% end %> +<% unless @statuses.empty? %> +<%= render 'new_trending_statuses' unless @statuses.empty? %> +<% end %> diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml index 6826c3b587..e97c493fe8 100644 --- a/app/views/application/_sidebar.html.haml +++ b/app/views/application/_sidebar.html.haml @@ -6,7 +6,7 @@ %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html') - if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - - trends = Trends.tags.get(true, 3) + - trends = Trends.tags.query.allowed.limit(3) - unless trends.empty? .endorsements-widget.trends-widget diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb index 084619cbd9..57f78170e4 100644 --- a/app/workers/scheduler/follow_recommendations_scheduler.rb +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -18,7 +18,7 @@ class Scheduler::FollowRecommendationsScheduler fallback_recommendations = FollowRecommendation.order(rank: :desc).limit(SET_SIZE) - I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq.each do |locale| + Trends.available_locales.each do |locale| recommendations = begin if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist FollowRecommendation.localized(locale).order(rank: :desc).limit(SET_SIZE).map { |recommendation| [recommendation.account_id, recommendation.rank] } @@ -49,11 +49,11 @@ class Scheduler::FollowRecommendationsScheduler end end - redis.pipelined do - redis.del(key(locale)) + redis.multi do |multi| + multi.del(key(locale)) recommendations.each do |(account_id, rank)| - redis.zadd(key(locale), rank, account_id) + multi.zadd(key(locale), rank, account_id) end end end diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 6ffe12ae06..c24146da41 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -7,7 +7,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/status.rb", - "line": 104, + "line": 105, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "result.joins(\"INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")", "render_path": null, @@ -20,6 +20,26 @@ "confidence": "Weak", "note": "" }, + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "30dfe36e87fe1b8f239df9a33d576e44a9863f73b680198d4713be6540ae61d3", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/trends/query.rb", + "line": 60, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "klass.joins(\"join unnest(array[#{ids.join(\",\")}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id\")", + "render_path": null, + "location": { + "type": "method", + "class": "Trends::Query", + "method": "to_arel" + }, + "user_input": "ids.join(\",\")", + "confidence": "Weak", + "note": "" + }, { "warning_type": "Redirect", "warning_code": 18, @@ -100,26 +120,6 @@ "confidence": "High", "note": "" }, - { - "warning_type": "SQL Injection", - "warning_code": 0, - "fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd", - "check_name": "SQL", - "message": "Possible SQL injection", - "file": "app/models/preview_card_filter.rb", - "line": 50, - "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")", - "render_path": null, - "location": { - "type": "method", - "class": "PreviewCardFilter", - "method": "trending_scope" - }, - "user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")", - "confidence": "Medium", - "note": "" - }, { "warning_type": "Cross-Site Scripting", "warning_code": 2, @@ -134,7 +134,7 @@ { "type": "template", "name": "admin/disputes/appeals/index", - "line": 16, + "line": 20, "file": "app/views/admin/disputes/appeals/index.html.haml", "rendered": { "name": "admin/disputes/appeals/_appeal", @@ -170,26 +170,6 @@ "confidence": "High", "note": "" }, - { - "warning_type": "SQL Injection", - "warning_code": 0, - "fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e", - "check_name": "SQL", - "message": "Possible SQL injection", - "file": "app/models/tag_filter.rb", - "line": 50, - "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")", - "render_path": null, - "location": { - "type": "method", - "class": "TagFilter", - "method": "trending_scope" - }, - "user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")", - "confidence": "Medium", - "note": "" - }, { "warning_type": "Cross-Site Scripting", "warning_code": 4, @@ -204,7 +184,7 @@ { "type": "template", "name": "admin/trends/links/index", - "line": 39, + "line": 45, "file": "app/views/admin/trends/links/index.html.haml", "rendered": { "name": "admin/trends/links/_preview_card", @@ -241,6 +221,6 @@ "note": "" } ], - "updated": "2022-02-13 02:24:12 +0100", + "updated": "2022-02-15 03:48:53 +0100", "brakeman_version": "5.2.1" } diff --git a/config/locales/en.yml b/config/locales/en.yml index f045174a9b..60c2915407 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -787,6 +787,15 @@ en: rejected: Links from this publisher won't trend title: Publishers rejected: Rejected + statuses: + allow: Allow post + allow_account: Allow author + disallow: Disallow post + disallow_account: Disallow author + shared_by: + one: Shared or favourited one time + other: Shared and favourited %{friendly_count} times + title: Trending posts tags: current_score: Current score %{score} dashboard: @@ -835,16 +844,21 @@ en: body: "%{reporter} has reported %{target}" body_remote: Someone from %{domain} has reported %{target} subject: New report for %{instance} (#%{id}) - new_trending_links: - body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated. - no_approved_links: There are currently no approved trending links. - requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}. - subject: New trending links up for review on %{instance} - new_trending_tags: - body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:' - no_approved_tags: There are currently no approved trending hashtags. - requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.' - subject: New trending hashtags up for review on %{instance} + new_trends: + body: 'The following items need a review before they can be displayed publicly:' + new_trending_links: + no_approved_links: There are currently no approved trending links. + requirements: 'Any of these candidates could surpass the #%{rank} approved trending link, which is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.' + title: Trending links + new_trending_statuses: + no_approved_statuses: There are currently no approved trending posts. + requirements: 'Any of these candidates could surpass the #%{rank} approved trending post, which is currently %{lowest_status_url} with a score of %{lowest_status_score}.' + title: Trending posts + new_trending_tags: + no_approved_tags: There are currently no approved trending hashtags. + requirements: 'Any of these candidates could surpass the #%{rank} approved trending hashtag, which is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.' + title: Trending hashtags + subject: New trends up for review on %{instance} aliases: add_new: Create alias created_msg: Successfully created a new alias. You can now initiate the move from the old account. diff --git a/config/navigation.rb b/config/navigation.rb index 3fc3747d59..620f78c579 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -34,6 +34,7 @@ SimpleNavigation::Configuration.run do |navigation| n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? } n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s| + s.item :statuses, safe_join([fa_icon('comments-o fw'), t('admin.trends.statuses.title')]), admin_trends_statuses_path, highlights_on: %r{/admin/trends/statuses} s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags} s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links} end diff --git a/config/routes.rb b/config/routes.rb index 176438e452..a820f32adf 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -327,6 +327,12 @@ Rails.application.routes.draw do end end + resources :statuses, only: [:index] do + collection do + post :batch + end + end + namespace :links do resources :preview_card_providers, only: [:index], path: :publishers do collection do @@ -448,6 +454,7 @@ Rails.application.routes.draw do namespace :trends do resources :links, only: [:index] resources :tags, only: [:index] + resources :statuses, only: [:index] end namespace :emails do @@ -554,6 +561,8 @@ Rails.application.routes.draw do namespace :trends do resources :tags, only: [:index] + resources :links, only: [:index] + resources :statuses, only: [:index] end post :measures, to: 'measures#create' diff --git a/db/migrate/20220202200743_add_trendable_to_accounts.rb b/db/migrate/20220202200743_add_trendable_to_accounts.rb new file mode 100644 index 0000000000..414df5108d --- /dev/null +++ b/db/migrate/20220202200743_add_trendable_to_accounts.rb @@ -0,0 +1,7 @@ +class AddTrendableToAccounts < ActiveRecord::Migration[6.1] + def change + add_column :accounts, :trendable, :boolean + add_column :accounts, :reviewed_at, :datetime + add_column :accounts, :requested_review_at, :datetime + end +end diff --git a/db/migrate/20220202200926_add_trendable_to_statuses.rb b/db/migrate/20220202200926_add_trendable_to_statuses.rb new file mode 100644 index 0000000000..7f38c8ca71 --- /dev/null +++ b/db/migrate/20220202200926_add_trendable_to_statuses.rb @@ -0,0 +1,5 @@ +class AddTrendableToStatuses < ActiveRecord::Migration[6.1] + def change + add_column :statuses, :trendable, :boolean + end +end diff --git a/db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb b/db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb new file mode 100644 index 0000000000..d5d995ecef --- /dev/null +++ b/db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveTrustLevelFromAccounts < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + safety_assured { remove_column :accounts, :trust_level, :integer } + end +end diff --git a/db/schema.rb b/db/schema.rb index 0e9b6e6191..e54de5b370 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -177,13 +177,15 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do t.string "also_known_as", array: true t.datetime "silenced_at" t.datetime "suspended_at" - t.integer "trust_level" t.boolean "hide_collections" t.integer "avatar_storage_schema_version" t.integer "header_storage_schema_version" t.string "devices_url" t.integer "suspension_origin" t.datetime "sensitized_at" + t.boolean "trendable" + t.datetime "reviewed_at" + t.datetime "requested_review_at" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id" @@ -887,6 +889,7 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do t.bigint "poll_id" t.datetime "deleted_at" t.datetime "edited_at" + t.boolean "trendable" t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)" t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)" t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))" @@ -1228,5 +1231,4 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do ORDER BY (sum(t0.rank)) DESC; SQL add_index "follow_recommendations", ["account_id"], name: "index_follow_recommendations_on_account_id", unique: true - end diff --git a/spec/controllers/api/v1/trends/tags_controller_spec.rb b/spec/controllers/api/v1/trends/tags_controller_spec.rb index e2e26dcab9..d29551c56c 100644 --- a/spec/controllers/api/v1/trends/tags_controller_spec.rb +++ b/spec/controllers/api/v1/trends/tags_controller_spec.rb @@ -7,10 +7,9 @@ RSpec.describe Api::V1::Trends::TagsController, type: :controller do describe 'GET #index' do before do - trending_tags = double() - - allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag)) - allow(Trends).to receive(:tags).and_return(trending_tags) + Fabricate.times(10, :tag).each do |tag| + 10.times { |i| Trends.tags.add(tag, i) } + end get :index end diff --git a/spec/mailers/previews/admin_mailer_preview.rb b/spec/mailers/previews/admin_mailer_preview.rb index 9c0372b47f..01436ba7a0 100644 --- a/spec/mailers/previews/admin_mailer_preview.rb +++ b/spec/mailers/previews/admin_mailer_preview.rb @@ -6,14 +6,9 @@ class AdminMailerPreview < ActionMailer::Preview AdminMailer.new_pending_account(Account.first, User.pending.first) end - # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags - def new_trending_tags - AdminMailer.new_trending_tags(Account.first, Tag.limit(3)) - end - - # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links - def new_trending_links - AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3)) + # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trends + def new_trends + AdminMailer.new_trends(Account.first, PreviewCard.limit(3), Tag.limit(3), Status.where(reblog_of_id: nil).limit(3)) end # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_appeal diff --git a/spec/models/trends/statuses_spec.rb b/spec/models/trends/statuses_spec.rb new file mode 100644 index 0000000000..9cc67acbe1 --- /dev/null +++ b/spec/models/trends/statuses_spec.rb @@ -0,0 +1,110 @@ +require 'rails_helper' + +RSpec.describe Trends::Statuses do + subject! { described_class.new(threshold: 5, review_threshold: 10, score_halflife: 8.hours) } + + let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) } + + describe 'Trends::Statuses::Query' do + let!(:query) { subject.query } + let!(:today) { at_time } + + let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: today) } + let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) } + + before do + 15.times { reblog(status1, today) } + 12.times { reblog(status2, today) } + + subject.refresh(today) + end + + describe '#filtered_for' do + let(:account) { Fabricate(:account) } + + it 'returns a composable query scope' do + expect(query.filtered_for(account)).to be_a Trends::Query + end + + it 'filters out blocked accounts' do + account.block!(status1.account) + expect(query.filtered_for(account).to_a).to eq [status2] + end + + it 'filters out muted accounts' do + account.mute!(status2.account) + expect(query.filtered_for(account).to_a).to eq [status1] + end + + it 'filters out blocked-by accounts' do + status1.account.block!(account) + expect(query.filtered_for(account).to_a).to eq [status2] + end + end + end + + describe '#add' do + let(:status) { Fabricate(:status) } + + before do + subject.add(status, 1, at_time) + end + + it 'records use' do + expect(subject.send(:recently_used_ids, at_time)).to eq [status.id] + end + end + + describe '#query' do + it 'returns a composable query scope' do + expect(subject.query).to be_a Trends::Query + end + + it 'responds to filtered_for' do + expect(subject.query).to respond_to(:filtered_for) + end + end + + describe '#refresh' do + let!(:today) { at_time } + let!(:yesterday) { today - 1.day } + + let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: yesterday) } + let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) } + let!(:status3) { Fabricate(:status, text: 'Baz', trendable: true, created_at: today) } + + before do + 13.times { reblog(status1, today) } + 13.times { reblog(status2, today) } + 4.times { reblog(status3, today) } + end + + context do + before do + subject.refresh(today) + end + + it 'calculates and re-calculates scores' do + expect(subject.query.limit(10).to_a).to eq [status2, status1] + end + + it 'omits statuses below threshold' do + expect(subject.query.limit(10).to_a).to_not include(status3) + end + end + + it 'decays scores' do + subject.refresh(today) + original_score = subject.score(status2.id) + expect(original_score).to be_a Float + subject.refresh(today + subject.options[:score_halflife]) + decayed_score = subject.score(status2.id) + expect(decayed_score).to be <= original_score / 2 + end + end + + def reblog(status, at_time) + reblog = Fabricate(:status, reblog: status, created_at: at_time) + subject.add(status, reblog.account_id, at_time) + end +end diff --git a/spec/models/trends/tags_spec.rb b/spec/models/trends/tags_spec.rb index 4f98c6aa4c..f48c735035 100644 --- a/spec/models/trends/tags_spec.rb +++ b/spec/models/trends/tags_spec.rb @@ -21,7 +21,7 @@ RSpec.describe Trends::Tags do end end - describe '#get' do + describe '#query' do pending end @@ -47,11 +47,11 @@ RSpec.describe Trends::Tags do end it 'calculates and re-calculates scores' do - expect(subject.get(false, 10)).to eq [tag1, tag3] + expect(subject.query.limit(10).to_a).to eq [tag1, tag3] end it 'omits hashtags below threshold' do - expect(subject.get(false, 10)).to_not include(tag2) + expect(subject.query.limit(10).to_a).to_not include(tag2) end end From d4592bbfcd091c4eaef8c8f24c47d5c2ce1bacd3 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 25 Feb 2022 00:34:33 +0100 Subject: [PATCH 04/24] Add explore page to web UI (#17123) * Add explore page to web UI * Fix not removing loaded statuses from trends on mute/block action --- app/javascript/mastodon/actions/trends.js | 91 +++++++++++-- app/javascript/mastodon/components/hashtag.js | 2 +- .../mastodon/components/status_action_bar.js | 7 +- .../mastodon/components/status_list.js | 3 + .../features/explore/components/story.js | 51 ++++++++ .../mastodon/features/explore/index.js | 91 +++++++++++++ .../mastodon/features/explore/links.js | 48 +++++++ .../mastodon/features/explore/results.js | 113 ++++++++++++++++ .../mastodon/features/explore/statuses.js | 48 +++++++ .../mastodon/features/explore/suggestions.js | 40 ++++++ .../mastodon/features/explore/tags.js | 40 ++++++ .../containers/trends_container.js | 6 +- .../mastodon/features/search/index.js | 17 --- .../features/ui/components/columns_area.js | 2 +- .../ui/components/navigation_panel.js | 1 + .../features/ui/components/tabs_bar.js | 6 +- app/javascript/mastodon/features/ui/index.js | 4 +- .../features/ui/util/async-components.js | 8 +- app/javascript/mastodon/reducers/search.js | 23 +++- .../mastodon/reducers/status_lists.js | 23 ++++ app/javascript/mastodon/reducers/trends.js | 43 ++++-- .../styles/mastodon/components.scss | 123 ++++++++++++++++++ 22 files changed, 727 insertions(+), 63 deletions(-) create mode 100644 app/javascript/mastodon/features/explore/components/story.js create mode 100644 app/javascript/mastodon/features/explore/index.js create mode 100644 app/javascript/mastodon/features/explore/links.js create mode 100644 app/javascript/mastodon/features/explore/results.js create mode 100644 app/javascript/mastodon/features/explore/statuses.js create mode 100644 app/javascript/mastodon/features/explore/suggestions.js create mode 100644 app/javascript/mastodon/features/explore/tags.js delete mode 100644 app/javascript/mastodon/features/search/index.js diff --git a/app/javascript/mastodon/actions/trends.js b/app/javascript/mastodon/actions/trends.js index 853e4f60ae..304bbebefb 100644 --- a/app/javascript/mastodon/actions/trends.js +++ b/app/javascript/mastodon/actions/trends.js @@ -1,31 +1,94 @@ import api from '../api'; +import { importFetchedStatuses } from './importer'; -export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST'; -export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS'; -export const TRENDS_FETCH_FAIL = 'TRENDS_FETCH_FAIL'; +export const TRENDS_TAGS_FETCH_REQUEST = 'TRENDS_TAGS_FETCH_REQUEST'; +export const TRENDS_TAGS_FETCH_SUCCESS = 'TRENDS_TAGS_FETCH_SUCCESS'; +export const TRENDS_TAGS_FETCH_FAIL = 'TRENDS_TAGS_FETCH_FAIL'; -export const fetchTrends = () => (dispatch, getState) => { - dispatch(fetchTrendsRequest()); +export const TRENDS_LINKS_FETCH_REQUEST = 'TRENDS_LINKS_FETCH_REQUEST'; +export const TRENDS_LINKS_FETCH_SUCCESS = 'TRENDS_LINKS_FETCH_SUCCESS'; +export const TRENDS_LINKS_FETCH_FAIL = 'TRENDS_LINKS_FETCH_FAIL'; + +export const TRENDS_STATUSES_FETCH_REQUEST = 'TRENDS_STATUSES_FETCH_REQUEST'; +export const TRENDS_STATUSES_FETCH_SUCCESS = 'TRENDS_STATUSES_FETCH_SUCCESS'; +export const TRENDS_STATUSES_FETCH_FAIL = 'TRENDS_STATUSES_FETCH_FAIL'; + +export const fetchTrendingHashtags = () => (dispatch, getState) => { + dispatch(fetchTrendingHashtagsRequest()); + + api(getState) + .get('/api/v1/trends/tags') + .then(({ data }) => dispatch(fetchTrendingHashtagsSuccess(data))) + .catch(err => dispatch(fetchTrendingHashtagsFail(err))); +}; + +export const fetchTrendingHashtagsRequest = () => ({ + type: TRENDS_TAGS_FETCH_REQUEST, + skipLoading: true, +}); + +export const fetchTrendingHashtagsSuccess = trends => ({ + type: TRENDS_TAGS_FETCH_SUCCESS, + trends, + skipLoading: true, +}); + +export const fetchTrendingHashtagsFail = error => ({ + type: TRENDS_TAGS_FETCH_FAIL, + error, + skipLoading: true, + skipAlert: true, +}); + +export const fetchTrendingLinks = () => (dispatch, getState) => { + dispatch(fetchTrendingLinksRequest()); api(getState) - .get('/api/v1/trends') - .then(({ data }) => dispatch(fetchTrendsSuccess(data))) - .catch(err => dispatch(fetchTrendsFail(err))); + .get('/api/v1/trends/links') + .then(({ data }) => dispatch(fetchTrendingLinksSuccess(data))) + .catch(err => dispatch(fetchTrendingLinksFail(err))); }; -export const fetchTrendsRequest = () => ({ - type: TRENDS_FETCH_REQUEST, +export const fetchTrendingLinksRequest = () => ({ + type: TRENDS_LINKS_FETCH_REQUEST, skipLoading: true, }); -export const fetchTrendsSuccess = trends => ({ - type: TRENDS_FETCH_SUCCESS, +export const fetchTrendingLinksSuccess = trends => ({ + type: TRENDS_LINKS_FETCH_SUCCESS, trends, skipLoading: true, }); -export const fetchTrendsFail = error => ({ - type: TRENDS_FETCH_FAIL, +export const fetchTrendingLinksFail = error => ({ + type: TRENDS_LINKS_FETCH_FAIL, + error, + skipLoading: true, + skipAlert: true, +}); + +export const fetchTrendingStatuses = () => (dispatch, getState) => { + dispatch(fetchTrendingStatusesRequest()); + + api(getState).get('/api/v1/trends/statuses').then(({ data }) => { + dispatch(importFetchedStatuses(data)); + dispatch(fetchTrendingStatusesSuccess(data)); + }).catch(err => dispatch(fetchTrendingStatusesFail(err))); +}; + +export const fetchTrendingStatusesRequest = () => ({ + type: TRENDS_STATUSES_FETCH_REQUEST, + skipLoading: true, +}); + +export const fetchTrendingStatusesSuccess = statuses => ({ + type: TRENDS_STATUSES_FETCH_SUCCESS, + statuses, + skipLoading: true, +}); + +export const fetchTrendingStatusesFail = error => ({ + type: TRENDS_STATUSES_FETCH_FAIL, error, skipLoading: true, skipAlert: true, diff --git a/app/javascript/mastodon/components/hashtag.js b/app/javascript/mastodon/components/hashtag.js index a793a32f54..7f442d189c 100644 --- a/app/javascript/mastodon/components/hashtag.js +++ b/app/javascript/mastodon/components/hashtag.js @@ -38,7 +38,7 @@ class SilentErrorBoundary extends React.Component { * * @type {(displayNumber: JSX.Element, pluralReady: number) => JSX.Element} */ -const accountsCountRenderer = (displayNumber, pluralReady) => ( +export const accountsCountRenderer = (displayNumber, pluralReady) => ( - - + + {shareButton} diff --git a/app/javascript/mastodon/components/status_list.js b/app/javascript/mastodon/components/status_list.js index eaaffcc3a8..35e5749a33 100644 --- a/app/javascript/mastodon/components/status_list.js +++ b/app/javascript/mastodon/components/status_list.js @@ -24,6 +24,7 @@ export default class StatusList extends ImmutablePureComponent { prepend: PropTypes.node, emptyMessage: PropTypes.node, alwaysPrepend: PropTypes.bool, + withCounters: PropTypes.bool, timelineId: PropTypes.string, }; @@ -100,6 +101,7 @@ export default class StatusList extends ImmutablePureComponent { contextType={timelineId} scrollKey={this.props.scrollKey} showThread + withCounters={this.props.withCounters} /> )) ) : null; @@ -114,6 +116,7 @@ export default class StatusList extends ImmutablePureComponent { onMoveDown={this.handleMoveDown} contextType={timelineId} showThread + withCounters={this.props.withCounters} /> )).concat(scrollableContent); } diff --git a/app/javascript/mastodon/features/explore/components/story.js b/app/javascript/mastodon/features/explore/components/story.js new file mode 100644 index 0000000000..5631280297 --- /dev/null +++ b/app/javascript/mastodon/features/explore/components/story.js @@ -0,0 +1,51 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import Blurhash from 'mastodon/components/blurhash'; +import { accountsCountRenderer } from 'mastodon/components/hashtag'; +import ShortNumber from 'mastodon/components/short_number'; +import Skeleton from 'mastodon/components/skeleton'; +import classNames from 'classnames'; + +export default class Story extends React.PureComponent { + + static propTypes = { + url: PropTypes.string, + title: PropTypes.string, + publisher: PropTypes.string, + sharedTimes: PropTypes.number, + thumbnail: PropTypes.string, + blurhash: PropTypes.string, + }; + + state = { + thumbnailLoaded: false, + }; + + handleImageLoad = () => this.setState({ thumbnailLoaded: true }); + + render () { + const { url, title, publisher, sharedTimes, thumbnail, blurhash } = this.props; + + const { thumbnailLoaded } = this.state; + + return ( + +
+
{publisher ? publisher : }
+
{title ? title : }
+
{typeof sharedTimes === 'number' ? : }
+
+ +
+ {thumbnail ? ( + +
+ +
+ ) : } +
+
+ ); + } + +} diff --git a/app/javascript/mastodon/features/explore/index.js b/app/javascript/mastodon/features/explore/index.js new file mode 100644 index 0000000000..ddacf5812b --- /dev/null +++ b/app/javascript/mastodon/features/explore/index.js @@ -0,0 +1,91 @@ +import React from 'react'; +import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; +import { connect } from 'react-redux'; +import PropTypes from 'prop-types'; +import Column from 'mastodon/components/column'; +import ColumnHeader from 'mastodon/components/column_header'; +import { NavLink, Switch, Route } from 'react-router-dom'; +import Links from './links'; +import Tags from './tags'; +import Statuses from './statuses'; +import Suggestions from './suggestions'; +import Search from 'mastodon/features/compose/containers/search_container'; +import SearchResults from './results'; + +const messages = defineMessages({ + title: { id: 'explore.title', defaultMessage: 'Explore' }, + searchResults: { id: 'explore.search_results', defaultMessage: 'Search results' }, +}); + +const mapStateToProps = state => ({ + layout: state.getIn(['meta', 'layout']), + isSearching: state.getIn(['search', 'submitted']), +}); + +export default @connect(mapStateToProps) +@injectIntl +class Explore extends React.PureComponent { + + static contextTypes = { + router: PropTypes.object, + }; + + static propTypes = { + intl: PropTypes.object.isRequired, + multiColumn: PropTypes.bool, + isSearching: PropTypes.bool, + layout: PropTypes.string, + }; + + handleHeaderClick = () => { + this.column.scrollTop(); + } + + setRef = c => { + this.column = c; + } + + render () { + const { intl, multiColumn, isSearching, layout } = this.props; + + return ( + + {layout === 'mobile' ? ( +
+ +
+ ) : ( + + )} + +
+ {isSearching ? ( + + ) : ( + +
+ + + + +
+ + + + + + + +
+ )} +
+
+ ); + } + +} diff --git a/app/javascript/mastodon/features/explore/links.js b/app/javascript/mastodon/features/explore/links.js new file mode 100644 index 0000000000..6649fb6e47 --- /dev/null +++ b/app/javascript/mastodon/features/explore/links.js @@ -0,0 +1,48 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import Story from './components/story'; +import LoadingIndicator from 'mastodon/components/loading_indicator'; +import { connect } from 'react-redux'; +import { fetchTrendingLinks } from 'mastodon/actions/trends'; + +const mapStateToProps = state => ({ + links: state.getIn(['trends', 'links', 'items']), + isLoading: state.getIn(['trends', 'links', 'isLoading']), +}); + +export default @connect(mapStateToProps) +class Links extends React.PureComponent { + + static propTypes = { + links: ImmutablePropTypes.list, + isLoading: PropTypes.bool, + dispatch: PropTypes.func.isRequired, + }; + + componentDidMount () { + const { dispatch } = this.props; + dispatch(fetchTrendingLinks()); + } + + render () { + const { isLoading, links } = this.props; + + return ( +
+ {isLoading ? () : links.map(link => ( + + ))} +
+ ); + } + +} diff --git a/app/javascript/mastodon/features/explore/results.js b/app/javascript/mastodon/features/explore/results.js new file mode 100644 index 0000000000..27e8aaa4f2 --- /dev/null +++ b/app/javascript/mastodon/features/explore/results.js @@ -0,0 +1,113 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import { FormattedMessage } from 'react-intl'; +import { connect } from 'react-redux'; +import { expandSearch } from 'mastodon/actions/search'; +import Account from 'mastodon/containers/account_container'; +import Status from 'mastodon/containers/status_container'; +import { ImmutableHashtag as Hashtag } from 'mastodon/components/hashtag'; +import { List as ImmutableList } from 'immutable'; +import LoadMore from 'mastodon/components/load_more'; +import LoadingIndicator from 'mastodon/components/loading_indicator'; + +const mapStateToProps = state => ({ + isLoading: state.getIn(['search', 'isLoading']), + results: state.getIn(['search', 'results']), +}); + +const appendLoadMore = (id, list, onLoadMore) => { + if (list.size >= 5) { + return list.push(); + } else { + return list; + } +}; + +const renderAccounts = (results, onLoadMore) => appendLoadMore('accounts', results.get('accounts').map(item => ( + +)), onLoadMore); + +const renderHashtags = (results, onLoadMore) => appendLoadMore('hashtags', results.get('hashtags').map(item => ( + +)), onLoadMore); + +const renderStatuses = (results, onLoadMore) => appendLoadMore('statuses', results.get('statuses').map(item => ( + +)), onLoadMore); + +export default @connect(mapStateToProps) +class Results extends React.PureComponent { + + static propTypes = { + results: ImmutablePropTypes.map, + isLoading: PropTypes.bool, + multiColumn: PropTypes.bool, + dispatch: PropTypes.func.isRequired, + }; + + state = { + type: 'all', + }; + + handleSelectAll = () => this.setState({ type: 'all' }); + handleSelectAccounts = () => this.setState({ type: 'accounts' }); + handleSelectHashtags = () => this.setState({ type: 'hashtags' }); + handleSelectStatuses = () => this.setState({ type: 'statuses' }); + handleLoadMoreAccounts = () => this.loadMore('accounts'); + handleLoadMoreStatuses = () => this.loadMore('statuses'); + handleLoadMoreHashtags = () => this.loadMore('hashtags'); + + loadMore (type) { + const { dispatch } = this.props; + dispatch(expandSearch(type)); + } + + render () { + const { isLoading, results } = this.props; + const { type } = this.state; + + let filteredResults = ImmutableList(); + + if (!isLoading) { + switch(type) { + case 'all': + filteredResults = filteredResults.concat(renderAccounts(results, this.handleLoadMoreAccounts), renderHashtags(results, this.handleLoadMoreHashtags), renderStatuses(results, this.handleLoadMoreStatuses)); + break; + case 'accounts': + filteredResults = filteredResults.concat(renderAccounts(results, this.handleLoadMoreAccounts)); + break; + case 'hashtags': + filteredResults = filteredResults.concat(renderHashtags(results, this.handleLoadMoreHashtags)); + break; + case 'statuses': + filteredResults = filteredResults.concat(renderStatuses(results, this.handleLoadMoreStatuses)); + break; + } + + if (filteredResults.size === 0) { + filteredResults = ( +
+ +
+ ); + } + } + + return ( + +
+ + + + +
+ +
+ {isLoading ? () : filteredResults} +
+
+ ); + } + +} diff --git a/app/javascript/mastodon/features/explore/statuses.js b/app/javascript/mastodon/features/explore/statuses.js new file mode 100644 index 0000000000..4e5530d84c --- /dev/null +++ b/app/javascript/mastodon/features/explore/statuses.js @@ -0,0 +1,48 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import StatusList from 'mastodon/components/status_list'; +import { FormattedMessage } from 'react-intl'; +import { connect } from 'react-redux'; +import { fetchTrendingStatuses } from 'mastodon/actions/trends'; + +const mapStateToProps = state => ({ + statusIds: state.getIn(['status_lists', 'trending', 'items']), + isLoading: state.getIn(['status_lists', 'trending', 'isLoading'], true), +}); + +export default @connect(mapStateToProps) +class Statuses extends React.PureComponent { + + static propTypes = { + statusIds: ImmutablePropTypes.list, + isLoading: PropTypes.bool, + multiColumn: PropTypes.bool, + dispatch: PropTypes.func.isRequired, + }; + + componentDidMount () { + const { dispatch } = this.props; + dispatch(fetchTrendingStatuses()); + } + + render () { + const { isLoading, statusIds, multiColumn } = this.props; + + const emptyMessage = ; + + return ( + + ); + } + +} diff --git a/app/javascript/mastodon/features/explore/suggestions.js b/app/javascript/mastodon/features/explore/suggestions.js new file mode 100644 index 0000000000..c094a8d934 --- /dev/null +++ b/app/javascript/mastodon/features/explore/suggestions.js @@ -0,0 +1,40 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import Account from 'mastodon/containers/account_container'; +import LoadingIndicator from 'mastodon/components/loading_indicator'; +import { connect } from 'react-redux'; +import { fetchSuggestions } from 'mastodon/actions/suggestions'; + +const mapStateToProps = state => ({ + suggestions: state.getIn(['suggestions', 'items']), + isLoading: state.getIn(['suggestions', 'isLoading']), +}); + +export default @connect(mapStateToProps) +class Suggestions extends React.PureComponent { + + static propTypes = { + isLoading: PropTypes.bool, + suggestions: ImmutablePropTypes.list, + dispatch: PropTypes.func.isRequired, + }; + + componentDidMount () { + const { dispatch } = this.props; + dispatch(fetchSuggestions(true)); + } + + render () { + const { isLoading, suggestions } = this.props; + + return ( +
+ {isLoading ? () : suggestions.map(suggestion => ( + + ))} +
+ ); + } + +} diff --git a/app/javascript/mastodon/features/explore/tags.js b/app/javascript/mastodon/features/explore/tags.js new file mode 100644 index 0000000000..c0ad9fc6ec --- /dev/null +++ b/app/javascript/mastodon/features/explore/tags.js @@ -0,0 +1,40 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import { ImmutableHashtag as Hashtag } from 'mastodon/components/hashtag'; +import LoadingIndicator from 'mastodon/components/loading_indicator'; +import { connect } from 'react-redux'; +import { fetchTrendingHashtags } from 'mastodon/actions/trends'; + +const mapStateToProps = state => ({ + hashtags: state.getIn(['trends', 'tags', 'items']), + isLoadingHashtags: state.getIn(['trends', 'tags', 'isLoading']), +}); + +export default @connect(mapStateToProps) +class Tags extends React.PureComponent { + + static propTypes = { + hashtags: ImmutablePropTypes.list, + isLoading: PropTypes.bool, + dispatch: PropTypes.func.isRequired, + }; + + componentDidMount () { + const { dispatch } = this.props; + dispatch(fetchTrendingHashtags()); + } + + render () { + const { isLoading, hashtags } = this.props; + + return ( +
+ {isLoading ? () : hashtags.map(hashtag => ( + + ))} +
+ ); + } + +} diff --git a/app/javascript/mastodon/features/getting_started/containers/trends_container.js b/app/javascript/mastodon/features/getting_started/containers/trends_container.js index 7a52687808..a73832db75 100644 --- a/app/javascript/mastodon/features/getting_started/containers/trends_container.js +++ b/app/javascript/mastodon/features/getting_started/containers/trends_container.js @@ -1,13 +1,13 @@ import { connect } from 'react-redux'; -import { fetchTrends } from 'mastodon/actions/trends'; +import { fetchTrendingHashtags } from 'mastodon/actions/trends'; import Trends from '../components/trends'; const mapStateToProps = state => ({ - trends: state.getIn(['trends', 'items']), + trends: state.getIn(['trends', 'tags', 'items']), }); const mapDispatchToProps = dispatch => ({ - fetchTrends: () => dispatch(fetchTrends()), + fetchTrends: () => dispatch(fetchTrendingHashtags()), }); export default connect(mapStateToProps, mapDispatchToProps)(Trends); diff --git a/app/javascript/mastodon/features/search/index.js b/app/javascript/mastodon/features/search/index.js deleted file mode 100644 index 76bf70d4bd..0000000000 --- a/app/javascript/mastodon/features/search/index.js +++ /dev/null @@ -1,17 +0,0 @@ -import React from 'react'; -import SearchContainer from 'mastodon/features/compose/containers/search_container'; -import SearchResultsContainer from 'mastodon/features/compose/containers/search_results_container'; - -const Search = () => ( -
- - -
-
- -
-
-
-); - -export default Search; diff --git a/app/javascript/mastodon/features/ui/components/columns_area.js b/app/javascript/mastodon/features/ui/components/columns_area.js index 193637113a..db047f5f0c 100644 --- a/app/javascript/mastodon/features/ui/components/columns_area.js +++ b/app/javascript/mastodon/features/ui/components/columns_area.js @@ -53,7 +53,7 @@ const messages = defineMessages({ publish: { id: 'compose_form.publish', defaultMessage: 'Toot' }, }); -const shouldHideFAB = path => path.match(/^\/statuses\/|^\/@[^/]+\/\d+|^\/publish|^\/search|^\/getting-started|^\/start/); +const shouldHideFAB = path => path.match(/^\/statuses\/|^\/@[^/]+\/\d+|^\/publish|^\/explore|^\/getting-started|^\/start/); export default @(component => injectIntl(component, { withRef: true })) class ColumnsArea extends ImmutablePureComponent { diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js index 901dbdfcb4..a70e5ab61c 100644 --- a/app/javascript/mastodon/features/ui/components/navigation_panel.js +++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js @@ -13,6 +13,7 @@ const NavigationPanel = () => ( + diff --git a/app/javascript/mastodon/features/ui/components/tabs_bar.js b/app/javascript/mastodon/features/ui/components/tabs_bar.js index a023bcf341..195403fd32 100644 --- a/app/javascript/mastodon/features/ui/components/tabs_bar.js +++ b/app/javascript/mastodon/features/ui/components/tabs_bar.js @@ -10,9 +10,9 @@ import NotificationsCounterIcon from './notifications_counter_icon'; export const links = [ , , - , - , - , + , + , + , , ]; diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js index 3feffa6564..2d0136992e 100644 --- a/app/javascript/mastodon/features/ui/index.js +++ b/app/javascript/mastodon/features/ui/index.js @@ -49,8 +49,8 @@ import { Mutes, PinnedStatuses, Lists, - Search, Directory, + Explore, FollowRecommendations, } from './util/async-components'; import { me } from '../../initial_state'; @@ -167,8 +167,8 @@ class SwitchingColumnsArea extends React.PureComponent { - + diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js index 5349bd6562..92c683e2f8 100644 --- a/app/javascript/mastodon/features/ui/util/async-components.js +++ b/app/javascript/mastodon/features/ui/util/async-components.js @@ -138,10 +138,6 @@ export function ListAdder () { return import(/*webpackChunkName: "features/list_adder" */'../../list_adder'); } -export function Search () { - return import(/*webpackChunkName: "features/search" */'../../search'); -} - export function Tesseract () { return import(/*webpackChunkName: "tesseract" */'tesseract.js'); } @@ -161,3 +157,7 @@ export function FollowRecommendations () { export function CompareHistoryModal () { return import(/*webpackChunkName: "modals/compare_history_modal" */'../components/compare_history_modal'); } + +export function Explore () { + return import(/* webpackChunkName: "features/explore" */'../../explore'); +} diff --git a/app/javascript/mastodon/reducers/search.js b/app/javascript/mastodon/reducers/search.js index 875b2d92b3..23bbe4d995 100644 --- a/app/javascript/mastodon/reducers/search.js +++ b/app/javascript/mastodon/reducers/search.js @@ -1,6 +1,8 @@ import { SEARCH_CHANGE, SEARCH_CLEAR, + SEARCH_FETCH_REQUEST, + SEARCH_FETCH_FAIL, SEARCH_FETCH_SUCCESS, SEARCH_SHOW, SEARCH_EXPAND_SUCCESS, @@ -17,6 +19,7 @@ const initialState = ImmutableMap({ submitted: false, hidden: false, results: ImmutableMap(), + isLoading: false, searchTerm: '', }); @@ -37,12 +40,22 @@ export default function search(state = initialState, action) { case COMPOSE_MENTION: case COMPOSE_DIRECT: return state.set('hidden', true); + case SEARCH_FETCH_REQUEST: + return state.set('isLoading', true); + case SEARCH_FETCH_FAIL: + return state.set('isLoading', false); case SEARCH_FETCH_SUCCESS: - return state.set('results', ImmutableMap({ - accounts: ImmutableList(action.results.accounts.map(item => item.id)), - statuses: ImmutableList(action.results.statuses.map(item => item.id)), - hashtags: fromJS(action.results.hashtags), - })).set('submitted', true).set('searchTerm', action.searchTerm); + return state.withMutations(map => { + map.set('results', ImmutableMap({ + accounts: ImmutableList(action.results.accounts.map(item => item.id)), + statuses: ImmutableList(action.results.statuses.map(item => item.id)), + hashtags: fromJS(action.results.hashtags), + })); + + map.set('submitted', true); + map.set('searchTerm', action.searchTerm); + map.set('isLoading', false); + }); case SEARCH_EXPAND_SUCCESS: const results = action.searchType === 'hashtags' ? fromJS(action.results.hashtags) : action.results[action.searchType].map(item => item.id); return state.updateIn(['results', action.searchType], list => list.concat(results)); diff --git a/app/javascript/mastodon/reducers/status_lists.js b/app/javascript/mastodon/reducers/status_lists.js index 9f8f28deed..49bc94a40d 100644 --- a/app/javascript/mastodon/reducers/status_lists.js +++ b/app/javascript/mastodon/reducers/status_lists.js @@ -17,6 +17,11 @@ import { import { PINNED_STATUSES_FETCH_SUCCESS, } from '../actions/pin_statuses'; +import { + TRENDS_STATUSES_FETCH_REQUEST, + TRENDS_STATUSES_FETCH_SUCCESS, + TRENDS_STATUSES_FETCH_FAIL, +} from '../actions/trends'; import { Map as ImmutableMap, List as ImmutableList } from 'immutable'; import { FAVOURITE_SUCCESS, @@ -26,6 +31,10 @@ import { PIN_SUCCESS, UNPIN_SUCCESS, } from '../actions/interactions'; +import { + ACCOUNT_BLOCK_SUCCESS, + ACCOUNT_MUTE_SUCCESS, +} from '../actions/accounts'; const initialState = ImmutableMap({ favourites: ImmutableMap({ @@ -43,6 +52,11 @@ const initialState = ImmutableMap({ loaded: false, items: ImmutableList(), }), + trending: ImmutableMap({ + next: null, + loaded: false, + items: ImmutableList(), + }), }); const normalizeList = (state, listType, statuses, next) => { @@ -96,6 +110,12 @@ export default function statusLists(state = initialState, action) { return normalizeList(state, 'bookmarks', action.statuses, action.next); case BOOKMARKED_STATUSES_EXPAND_SUCCESS: return appendToList(state, 'bookmarks', action.statuses, action.next); + case TRENDS_STATUSES_FETCH_REQUEST: + return state.setIn(['trending', 'isLoading'], true); + case TRENDS_STATUSES_FETCH_FAIL: + return state.setIn(['trending', 'isLoading'], false); + case TRENDS_STATUSES_FETCH_SUCCESS: + return normalizeList(state, 'trending', action.statuses, action.next); case FAVOURITE_SUCCESS: return prependOneToList(state, 'favourites', action.status); case UNFAVOURITE_SUCCESS: @@ -110,6 +130,9 @@ export default function statusLists(state = initialState, action) { return prependOneToList(state, 'pins', action.status); case UNPIN_SUCCESS: return removeOneFromList(state, 'pins', action.status); + case ACCOUNT_BLOCK_SUCCESS: + case ACCOUNT_MUTE_SUCCESS: + return state.updateIn(['trending', 'items'], ImmutableList(), list => list.filterNot(statusId => action.statuses.getIn([statusId, 'account']) === action.relationship.id)); default: return state; } diff --git a/app/javascript/mastodon/reducers/trends.js b/app/javascript/mastodon/reducers/trends.js index 5cecc8fcab..3e01bd07d0 100644 --- a/app/javascript/mastodon/reducers/trends.js +++ b/app/javascript/mastodon/reducers/trends.js @@ -1,22 +1,45 @@ -import { TRENDS_FETCH_REQUEST, TRENDS_FETCH_SUCCESS, TRENDS_FETCH_FAIL } from '../actions/trends'; +import { + TRENDS_TAGS_FETCH_REQUEST, + TRENDS_TAGS_FETCH_SUCCESS, + TRENDS_TAGS_FETCH_FAIL, + TRENDS_LINKS_FETCH_REQUEST, + TRENDS_LINKS_FETCH_SUCCESS, + TRENDS_LINKS_FETCH_FAIL, +} from 'mastodon/actions/trends'; import { Map as ImmutableMap, List as ImmutableList, fromJS } from 'immutable'; const initialState = ImmutableMap({ - items: ImmutableList(), - isLoading: false, + tags: ImmutableMap({ + items: ImmutableList(), + isLoading: false, + }), + + links: ImmutableMap({ + items: ImmutableList(), + isLoading: false, + }), }); export default function trendsReducer(state = initialState, action) { switch(action.type) { - case TRENDS_FETCH_REQUEST: - return state.set('isLoading', true); - case TRENDS_FETCH_SUCCESS: + case TRENDS_TAGS_FETCH_REQUEST: + return state.setIn(['tags', 'isLoading'], true); + case TRENDS_TAGS_FETCH_SUCCESS: + return state.withMutations(map => { + map.setIn(['tags', 'items'], fromJS(action.trends)); + map.setIn(['tags', 'isLoading'], false); + }); + case TRENDS_TAGS_FETCH_FAIL: + return state.setIn(['tags', 'isLoading'], false); + case TRENDS_LINKS_FETCH_REQUEST: + return state.setIn(['links', 'isLoading'], true); + case TRENDS_LINKS_FETCH_SUCCESS: return state.withMutations(map => { - map.set('items', fromJS(action.trends)); - map.set('isLoading', false); + map.setIn(['links', 'items'], fromJS(action.trends)); + map.setIn(['links', 'isLoading'], false); }); - case TRENDS_FETCH_FAIL: - return state.set('isLoading', false); + case TRENDS_LINKS_FETCH_FAIL: + return state.setIn(['links', 'isLoading'], false); default: return state; } diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 6d30bea83b..647e7ea31b 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -2797,6 +2797,10 @@ a.account__display-name { position: relative; min-height: 120px; } + + .scrollable { + flex: 1 1 auto; + } } .scrollable.fullscreen { @@ -7724,3 +7728,122 @@ noscript { text-align: center; } } + +.explore__search-header { + background: $ui-base-color; + display: flex; + align-items: flex-start; + justify-content: center; + padding: 15px; + + .search { + width: 100%; + margin-bottom: 0; + } + + .search__input { + border-radius: 4px; + color: $inverted-text-color; + background: $simple-background-color; + padding: 10px; + + &::placeholder { + color: $dark-text-color; + } + } + + .search .fa { + top: 10px; + right: 10px; + color: $dark-text-color; + } + + .search .fa-times-circle { + top: 12px; + } +} + +.explore__search-results { + flex: 1 1 auto; + display: flex; + flex-direction: column; +} + +.story { + display: flex; + align-items: center; + color: $primary-text-color; + text-decoration: none; + padding: 15px 0; + border-bottom: 1px solid lighten($ui-base-color, 8%); + + &:last-child { + border-bottom: 0; + } + + &:hover, + &:active, + &:focus { + background-color: lighten($ui-base-color, 4%); + } + + &__details { + padding: 0 15px; + flex: 1 1 auto; + + &__publisher { + color: $darker-text-color; + margin-bottom: 4px; + } + + &__title { + font-size: 19px; + line-height: 24px; + font-weight: 500; + margin-bottom: 4px; + } + + &__shared { + color: $darker-text-color; + } + } + + &__thumbnail { + flex: 0 0 auto; + margin: 0 15px; + position: relative; + width: 120px; + height: 120px; + + .skeleton { + width: 100%; + height: 100%; + } + + img { + border-radius: 4px; + display: block; + margin: 0; + width: 100%; + height: 100%; + object-fit: cover; + } + + &__preview { + border-radius: 4px; + display: block; + margin: 0; + width: 100%; + height: 100%; + object-fit: fill; + position: absolute; + top: 0; + left: 0; + z-index: 0; + + &--hidden { + display: none; + } + } + } +} From 2cd31b31778cec3b282a44f03a03844d92a4e8cc Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Feb 2022 00:51:01 +0100 Subject: [PATCH 05/24] Fix reply button on media modal not giving focus to compose form (#17626) * Avoid compose form and modal management fighting for focus * Fix reply button on media modal footer not giving focus to compose form --- app/javascript/mastodon/actions/modal.js | 3 +- .../mastodon/components/modal_root.js | 5 +++- .../compose/components/compose_form.js | 9 ++++-- .../picture_in_picture/components/footer.js | 2 +- .../features/ui/components/modal_root.js | 9 +++--- .../features/ui/containers/modal_container.js | 11 +++---- app/javascript/mastodon/reducers/modal.js | 30 +++++++++++++++---- 7 files changed, 50 insertions(+), 19 deletions(-) diff --git a/app/javascript/mastodon/actions/modal.js b/app/javascript/mastodon/actions/modal.js index 3d0299db58..3e576fab8e 100644 --- a/app/javascript/mastodon/actions/modal.js +++ b/app/javascript/mastodon/actions/modal.js @@ -9,9 +9,10 @@ export function openModal(type, props) { }; }; -export function closeModal(type) { +export function closeModal(type, options = { ignoreFocus: false }) { return { type: MODAL_CLOSE, modalType: type, + ignoreFocus: options.ignoreFocus, }; }; diff --git a/app/javascript/mastodon/components/modal_root.js b/app/javascript/mastodon/components/modal_root.js index 755c46fd6b..b894aeaf98 100644 --- a/app/javascript/mastodon/components/modal_root.js +++ b/app/javascript/mastodon/components/modal_root.js @@ -18,6 +18,7 @@ export default class ModalRoot extends React.PureComponent { g: PropTypes.number, b: PropTypes.number, }), + ignoreFocus: PropTypes.bool, }; activeElement = this.props.children ? document.activeElement : null; @@ -72,7 +73,9 @@ export default class ModalRoot extends React.PureComponent { // immediately selectable, we have to wait for observers to run, as // described in https://github.com/WICG/inert#performance-and-gotchas Promise.resolve().then(() => { - this.activeElement.focus({ preventScroll: true }); + if (!this.props.ignoreFocus) { + this.activeElement.focus({ preventScroll: true }); + } this.activeElement = null; }).catch(console.error); diff --git a/app/javascript/mastodon/features/compose/components/compose_form.js b/app/javascript/mastodon/features/compose/components/compose_form.js index d75145a09e..d7635da404 100644 --- a/app/javascript/mastodon/features/compose/components/compose_form.js +++ b/app/javascript/mastodon/features/compose/components/compose_form.js @@ -163,8 +163,13 @@ class ComposeForm extends ImmutablePureComponent { selectionStart = selectionEnd; } - this.autosuggestTextarea.textarea.setSelectionRange(selectionStart, selectionEnd); - this.autosuggestTextarea.textarea.focus(); + // Because of the wicg-inert polyfill, the activeElement may not be + // immediately selectable, we have to wait for observers to run, as + // described in https://github.com/WICG/inert#performance-and-gotchas + Promise.resolve().then(() => { + this.autosuggestTextarea.textarea.setSelectionRange(selectionStart, selectionEnd); + this.autosuggestTextarea.textarea.focus(); + }).catch(console.error); } else if(prevProps.isSubmitting && !this.props.isSubmitting) { this.autosuggestTextarea.textarea.focus(); } else if (this.props.spoiler !== prevProps.spoiler) { diff --git a/app/javascript/mastodon/features/picture_in_picture/components/footer.js b/app/javascript/mastodon/features/picture_in_picture/components/footer.js index 0de562ee1d..690a77531c 100644 --- a/app/javascript/mastodon/features/picture_in_picture/components/footer.js +++ b/app/javascript/mastodon/features/picture_in_picture/components/footer.js @@ -60,7 +60,7 @@ class Footer extends ImmutablePureComponent { const { router } = this.context; if (onClose) { - onClose(); + onClose(true); } dispatch(replyCompose(status, router.history)); diff --git a/app/javascript/mastodon/features/ui/components/modal_root.js b/app/javascript/mastodon/features/ui/components/modal_root.js index 7b14fe5caa..3fc235849d 100644 --- a/app/javascript/mastodon/features/ui/components/modal_root.js +++ b/app/javascript/mastodon/features/ui/components/modal_root.js @@ -45,6 +45,7 @@ export default class ModalRoot extends React.PureComponent { type: PropTypes.string, props: PropTypes.object, onClose: PropTypes.func.isRequired, + ignoreFocus: PropTypes.bool, }; state = { @@ -79,7 +80,7 @@ export default class ModalRoot extends React.PureComponent { return ; } - handleClose = () => { + handleClose = (ignoreFocus = false) => { const { onClose } = this.props; let message = null; try { @@ -89,7 +90,7 @@ export default class ModalRoot extends React.PureComponent { // isn't set. // This would be much smoother with react-intl 3+ and `forwardRef`. } - onClose(message); + onClose(message, ignoreFocus); } setModalRef = (c) => { @@ -97,12 +98,12 @@ export default class ModalRoot extends React.PureComponent { } render () { - const { type, props } = this.props; + const { type, props, ignoreFocus } = this.props; const { backgroundColor } = this.state; const visible = !!type; return ( - + {visible && ( {(SpecificComponent) => } diff --git a/app/javascript/mastodon/features/ui/containers/modal_container.js b/app/javascript/mastodon/features/ui/containers/modal_container.js index 34fec82066..35be262221 100644 --- a/app/javascript/mastodon/features/ui/containers/modal_container.js +++ b/app/javascript/mastodon/features/ui/containers/modal_container.js @@ -3,22 +3,23 @@ import { openModal, closeModal } from '../../../actions/modal'; import ModalRoot from '../components/modal_root'; const mapStateToProps = state => ({ - type: state.getIn(['modal', 0, 'modalType'], null), - props: state.getIn(['modal', 0, 'modalProps'], {}), + ignoreFocus: state.getIn(['modal', 'ignoreFocus']), + type: state.getIn(['modal', 'stack', 0, 'modalType'], null), + props: state.getIn(['modal', 'stack', 0, 'modalProps'], {}), }); const mapDispatchToProps = dispatch => ({ - onClose (confirmationMessage) { + onClose (confirmationMessage, ignoreFocus = false) { if (confirmationMessage) { dispatch( openModal('CONFIRM', { message: confirmationMessage.message, confirm: confirmationMessage.confirm, - onConfirm: () => dispatch(closeModal()), + onConfirm: () => dispatch(closeModal(undefined, { ignoreFocus })), }), ); } else { - dispatch(closeModal()); + dispatch(closeModal(undefined, { ignoreFocus })); } }, }); diff --git a/app/javascript/mastodon/reducers/modal.js b/app/javascript/mastodon/reducers/modal.js index 41161a2060..3eab07d9d4 100644 --- a/app/javascript/mastodon/reducers/modal.js +++ b/app/javascript/mastodon/reducers/modal.js @@ -3,16 +3,36 @@ import { TIMELINE_DELETE } from '../actions/timelines'; import { COMPOSE_UPLOAD_CHANGE_SUCCESS } from '../actions/compose'; import { Stack as ImmutableStack, Map as ImmutableMap } from 'immutable'; -export default function modal(state = ImmutableStack(), action) { +const initialState = ImmutableMap({ + ignoreFocus: false, + stack: ImmutableStack(), +}); + +const popModal = (state, { modalType, ignoreFocus }) => { + if (modalType === undefined || modalType === state.getIn(['stack', 0, 'modalType'])) { + return state.set('ignoreFocus', !!ignoreFocus).update('stack', stack => stack.shift()); + } else { + return state; + } +}; + +const pushModal = (state, modalType, modalProps) => { + return state.withMutations(map => { + map.set('ignoreFocus', false); + map.update('stack', stack => stack.unshift(ImmutableMap({ modalType, modalProps }))); + }); +}; + +export default function modal(state = initialState, action) { switch(action.type) { case MODAL_OPEN: - return state.unshift(ImmutableMap({ modalType: action.modalType, modalProps: action.modalProps })); + return pushModal(state, action.modalType, action.modalProps); case MODAL_CLOSE: - return (action.modalType === undefined || action.modalType === state.getIn([0, 'modalType'])) ? state.shift() : state; + return popModal(state, action); case COMPOSE_UPLOAD_CHANGE_SUCCESS: - return state.getIn([0, 'modalType']) === 'FOCAL_POINT' ? state.shift() : state; + return popModal(state, { modalType: 'FOCAL_POINT', ignoreFocus: false }); case TIMELINE_DELETE: - return state.filterNot((modal) => modal.get('modalProps').statusId === action.id); + return state.update('stack', stack => stack.filterNot((modal) => modal.get('modalProps').statusId === action.id)); default: return state; } From 21e80a979253e4c98fd8852c93c6069028c184f6 Mon Sep 17 00:00:00 2001 From: sasanquaneuf Date: Fri, 25 Feb 2022 09:16:52 +0900 Subject: [PATCH 06/24] Escape database passwords in config/database.yml (#17627) * Add double quotes for using passwords that start with a comma * Escape database password in yml --- config/database.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/database.yml b/config/database.yml index c10bff6b2e..9b8d096e90 100644 --- a/config/database.yml +++ b/config/database.yml @@ -9,7 +9,7 @@ development: <<: *default database: <%= ENV['DB_NAME'] || 'mastodon_development' %> username: <%= ENV['DB_USER'] %> - password: <%= ENV['DB_PASS'] %> + password: <%= (ENV['DB_PASS'] || '').to_json %> host: <%= ENV['DB_HOST'] %> port: <%= ENV['DB_PORT'] %> @@ -20,7 +20,7 @@ test: <<: *default database: <%= ENV['DB_NAME'] || 'mastodon' %>_test<%= ENV['TEST_ENV_NUMBER'] %> username: <%= ENV['DB_USER'] %> - password: <%= ENV['DB_PASS'] %> + password: <%= (ENV['DB_PASS'] || '').to_json %> host: <%= ENV['DB_HOST'] %> port: <%= ENV['DB_PORT'] %> @@ -28,7 +28,7 @@ production: <<: *default database: <%= ENV['DB_NAME'] || 'mastodon_production' %> username: <%= ENV['DB_USER'] || 'mastodon' %> - password: <%= ENV['DB_PASS'] || '' %> + password: <%= (ENV['DB_PASS'] || '').to_json %> host: <%= ENV['DB_HOST'] || 'localhost' %> port: <%= ENV['DB_PORT'] || 5432 %> prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %> From e884f7dfbd46f826717b420ae3ddb15dab7864ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Feb 2022 01:18:55 +0100 Subject: [PATCH 07/24] Bump nokogiri from 1.13.1 to 1.13.2 (#17604) Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.1 to 1.13.2. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.1...v1.13.2) --- updated-dependencies: - dependency-name: nokogiri dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2ad52feb02..a1af097cb7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -374,7 +374,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) mini_mime (1.1.2) - mini_portile2 (2.7.1) + mini_portile2 (2.8.0) minitest (5.15.0) msgpack (1.4.4) multi_json (1.15.0) @@ -384,8 +384,8 @@ GEM net-ssh (>= 2.6.5, < 7.0.0) net-ssh (6.1.0) nio4r (2.5.8) - nokogiri (1.13.1) - mini_portile2 (~> 2.7.0) + nokogiri (1.13.2) + mini_portile2 (~> 2.8.0) racc (~> 1.4) nsa (0.2.8) activesupport (>= 4.2, < 7) From 255748dff48b80335cfb7af12d1ea67979af09ad Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Feb 2022 01:20:41 +0100 Subject: [PATCH 08/24] =?UTF-8?q?Fix=20media=20modal=20footer's=20?= =?UTF-8?q?=E2=80=9Cexternal=20link=E2=80=9D=20not=20being=20a=20link=20(#?= =?UTF-8?q?17561)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mastodon/components/icon_button.js | 18 +++++++++++++++++- .../picture_in_picture/components/footer.js | 2 +- app/javascript/styles/mastodon/components.scss | 5 +++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/app/javascript/mastodon/components/icon_button.js b/app/javascript/mastodon/components/icon_button.js index 7ec39198a4..6a653675b8 100644 --- a/app/javascript/mastodon/components/icon_button.js +++ b/app/javascript/mastodon/components/icon_button.js @@ -27,6 +27,7 @@ export default class IconButton extends React.PureComponent { tabIndex: PropTypes.string, counter: PropTypes.number, obfuscateCount: PropTypes.bool, + href: PropTypes.string, }; static defaultProps = { @@ -102,6 +103,7 @@ export default class IconButton extends React.PureComponent { title, counter, obfuscateCount, + href, } = this.props; const { @@ -123,6 +125,20 @@ export default class IconButton extends React.PureComponent { style.width = 'auto'; } + let contents = ( + + + ); + + if (href) { + contents = ( + + {contents} + + ); + } + return ( ); } diff --git a/app/javascript/mastodon/features/picture_in_picture/components/footer.js b/app/javascript/mastodon/features/picture_in_picture/components/footer.js index 690a77531c..0cb42b25aa 100644 --- a/app/javascript/mastodon/features/picture_in_picture/components/footer.js +++ b/app/javascript/mastodon/features/picture_in_picture/components/footer.js @@ -156,7 +156,7 @@ class Footer extends ImmutablePureComponent { - {withOpenButton && } + {withOpenButton && } ); } diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 647e7ea31b..6b18ca6f21 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -166,6 +166,11 @@ transition-property: background-color, color; text-decoration: none; + a { + color: inherit; + text-decoration: none; + } + &:hover, &:active, &:focus { From 50452064b352555b9c404c7b186dd2f090017fad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Feb 2022 01:48:08 +0100 Subject: [PATCH 09/24] Bump nokogiri from 1.13.2 to 1.13.3 (#17641) Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.2 to 1.13.3. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.2...v1.13.3) --- updated-dependencies: - dependency-name: nokogiri dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index a1af097cb7..0ee0d4c9b2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -384,7 +384,7 @@ GEM net-ssh (>= 2.6.5, < 7.0.0) net-ssh (6.1.0) nio4r (2.5.8) - nokogiri (1.13.2) + nokogiri (1.13.3) mini_portile2 (~> 2.8.0) racc (~> 1.4) nsa (0.2.8) From 85f6a960f9965ef40ad6368588b6f7bf9d8936a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Feb 2022 01:48:46 +0100 Subject: [PATCH 10/24] Bump aws-sdk-s3 from 1.112.0 to 1.113.0 (#17642) Bumps [aws-sdk-s3](https://github.com/aws/aws-sdk-ruby) from 1.112.0 to 1.113.0. - [Release notes](https://github.com/aws/aws-sdk-ruby/releases) - [Changelog](https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-ruby/commits) --- updated-dependencies: - dependency-name: aws-sdk-s3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile | 2 +- Gemfile.lock | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 1a3df276e0..b5d15da619 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,7 @@ gem 'makara', '~> 0.5' gem 'pghero', '~> 2.8' gem 'dotenv-rails', '~> 2.7' -gem 'aws-sdk-s3', '~> 1.112', require: false +gem 'aws-sdk-s3', '~> 1.113', require: false gem 'fog-core', '<= 2.1.0' gem 'fog-openstack', '~> 0.3', require: false gem 'kt-paperclip', '~> 7.1' diff --git a/Gemfile.lock b/Gemfile.lock index 0ee0d4c9b2..7054e83736 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,17 +79,17 @@ GEM encryptor (~> 3.0.0) awrence (1.1.1) aws-eventstream (1.2.0) - aws-partitions (1.553.0) - aws-sdk-core (3.126.0) + aws-partitions (1.558.0) + aws-sdk-core (3.127.0) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.525.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-kms (1.54.0) - aws-sdk-core (~> 3, >= 3.126.0) + aws-sdk-kms (1.55.0) + aws-sdk-core (~> 3, >= 3.127.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.112.0) - aws-sdk-core (~> 3, >= 3.126.0) + aws-sdk-s3 (1.113.0) + aws-sdk-core (~> 3, >= 3.127.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.4) aws-sigv4 (1.4.0) @@ -303,7 +303,7 @@ GEM terminal-table (>= 1.5.1) idn-ruby (0.1.4) ipaddress (0.8.3) - jmespath (1.5.0) + jmespath (1.6.0) json (2.5.1) json-canonicalization (0.3.0) json-ld (3.2.0) @@ -683,7 +683,7 @@ DEPENDENCIES active_record_query_trace (~> 1.8) addressable (~> 2.8) annotate (~> 3.2) - aws-sdk-s3 (~> 1.112) + aws-sdk-s3 (~> 1.113) better_errors (~> 2.9) binding_of_caller (~> 1.0) blurhash (~> 0.1) From 0f75490b497d8d321bd1d4e779eaafdd6ec9b6c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Feb 2022 02:15:20 +0100 Subject: [PATCH 11/24] Bump pg from 1.3.2 to 1.3.3 (#17643) Bumps [pg](https://github.com/ged/ruby-pg) from 1.3.2 to 1.3.3. - [Release notes](https://github.com/ged/ruby-pg/releases) - [Changelog](https://github.com/ged/ruby-pg/blob/master/History.rdoc) - [Commits](https://github.com/ged/ruby-pg/compare/v1.3.2...v1.3.3) --- updated-dependencies: - dependency-name: pg dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7054e83736..48337d9ffe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -416,7 +416,7 @@ GEM parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - pg (1.3.2) + pg (1.3.3) pghero (2.8.2) activerecord (>= 5) pkg-config (1.4.7) From a5c24d5c4d75f3f3144f69c8f60f542707a82584 Mon Sep 17 00:00:00 2001 From: MitarashiDango Date: Sat, 26 Feb 2022 01:04:08 +0900 Subject: [PATCH 12/24] Fix unable to unpin follower-only posts (#17647) --- app/presenters/status_relationships_presenter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/presenters/status_relationships_presenter.rb b/app/presenters/status_relationships_presenter.rb index 70fb2ba900..4163bb098c 100644 --- a/app/presenters/status_relationships_presenter.rb +++ b/app/presenters/status_relationships_presenter.rb @@ -15,7 +15,7 @@ class StatusRelationshipsPresenter statuses = statuses.compact status_ids = statuses.flat_map { |s| [s.id, s.reblog_of_id] }.uniq.compact conversation_ids = statuses.filter_map(&:conversation_id).uniq - pinnable_status_ids = statuses.map(&:proper).filter_map { |s| s.id if s.account_id == current_account_id && %w(public unlisted).include?(s.visibility) } + pinnable_status_ids = statuses.map(&:proper).filter_map { |s| s.id if s.account_id == current_account_id && %w(public unlisted private).include?(s.visibility) } @reblogs_map = Status.reblogs_map(status_ids, current_account_id).merge(options[:reblogs_map] || {}) @favourites_map = Status.favourites_map(status_ids, current_account_id).merge(options[:favourites_map] || {}) From 59824aadcd19ba88cc5f29e3e0212383e4c2c653 Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 09:46:13 +0100 Subject: [PATCH 13/24] Adapt view for glitch-soc's theming system --- app/views/admin/email_domain_blocks/index.html.haml | 3 --- app/views/admin/email_domain_blocks/new.html.haml | 3 --- app/views/admin/trends/statuses/index.html.haml | 3 --- 3 files changed, 9 deletions(-) diff --git a/app/views/admin/email_domain_blocks/index.html.haml b/app/views/admin/email_domain_blocks/index.html.haml index b073e87162..9f16e0d5c3 100644 --- a/app/views/admin/email_domain_blocks/index.html.haml +++ b/app/views/admin/email_domain_blocks/index.html.haml @@ -4,9 +4,6 @@ - content_for :heading_actions do = link_to t('admin.email_domain_blocks.add_new'), new_admin_email_domain_block_path, class: 'button' -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = form_for(@form, url: batch_admin_email_domain_blocks_path) do |f| = hidden_field_tag :page, params[:page] || 1 diff --git a/app/views/admin/email_domain_blocks/new.html.haml b/app/views/admin/email_domain_blocks/new.html.haml index 524b699688..fa1d950ad2 100644 --- a/app/views/admin/email_domain_blocks/new.html.haml +++ b/app/views/admin/email_domain_blocks/new.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = simple_form_for @email_domain_block, url: admin_email_domain_blocks_path do |f| = render 'shared/error_messages', object: @email_domain_block diff --git a/app/views/admin/trends/statuses/index.html.haml b/app/views/admin/trends/statuses/index.html.haml index 3476882628..3166bc6c11 100644 --- a/app/views/admin/trends/statuses/index.html.haml +++ b/app/views/admin/trends/statuses/index.html.haml @@ -1,9 +1,6 @@ - content_for :page_title do = t('admin.trends.statuses.title') -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - = form_tag admin_trends_statuses_path, method: 'GET', class: 'simple_form' do - Trends::StatusFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? From 2e2b8a9a40ecdd2a7a63ecccb252af46179bbdb5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 25 Feb 2022 00:34:14 +0100 Subject: [PATCH 14/24] [Glitch] Add trending statuses Port SCSS changes from 27965ce5edff20db2de1dd233c88f8393bb0da0b Signed-off-by: Claire --- app/javascript/flavours/glitch/styles/accounts.scss | 10 ++++++++-- app/javascript/flavours/glitch/styles/tables.scss | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/javascript/flavours/glitch/styles/accounts.scss b/app/javascript/flavours/glitch/styles/accounts.scss index 56b143fe62..a3bfb05079 100644 --- a/app/javascript/flavours/glitch/styles/accounts.scss +++ b/app/javascript/flavours/glitch/styles/accounts.scss @@ -333,7 +333,8 @@ } .batch-table__row--muted .pending-account__header, -.batch-table__row--muted .accounts-table { +.batch-table__row--muted .accounts-table, +.batch-table__row--muted .name-tag { &, a, strong { @@ -341,6 +342,10 @@ } } +.batch-table__row--muted .name-tag .avatar { + opacity: 0.5; +} + .batch-table__row--muted .accounts-table { tbody td.accounts-table__extra, &__count, @@ -354,7 +359,8 @@ } .batch-table__row--attention .pending-account__header, -.batch-table__row--attention .accounts-table { +.batch-table__row--attention .accounts-table, +.batch-table__row--attention .name-tag { &, a, strong { diff --git a/app/javascript/flavours/glitch/styles/tables.scss b/app/javascript/flavours/glitch/styles/tables.scss index 12c84a6c90..8b5933b7b5 100644 --- a/app/javascript/flavours/glitch/styles/tables.scss +++ b/app/javascript/flavours/glitch/styles/tables.scss @@ -210,6 +210,7 @@ a.table-action-link { &__content { padding-top: 12px; padding-bottom: 16px; + overflow: hidden; &--unpadded { padding: 0; @@ -292,3 +293,9 @@ a.table-action-link { } } } + +.one-liner { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} From 8e04ba87b709365facbbe28682b2809225f209bc Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Feb 2022 00:51:01 +0100 Subject: [PATCH 15/24] [Glitch] Fix reply button on media modal not giving focus to compose form Port 2cd31b31778cec3b282a44f03a03844d92a4e8cc to glitch-soc Signed-off-by: Claire --- .../flavours/glitch/actions/modal.js | 3 +- .../flavours/glitch/components/modal_root.js | 5 +++- .../compose/components/compose_form.js | 11 +++++-- .../picture_in_picture/components/footer.js | 2 +- .../features/ui/components/modal_root.js | 9 +++--- .../features/ui/containers/modal_container.js | 11 +++---- .../flavours/glitch/reducers/modal.js | 30 +++++++++++++++---- 7 files changed, 51 insertions(+), 20 deletions(-) diff --git a/app/javascript/flavours/glitch/actions/modal.js b/app/javascript/flavours/glitch/actions/modal.js index 3d0299db58..3e576fab8e 100644 --- a/app/javascript/flavours/glitch/actions/modal.js +++ b/app/javascript/flavours/glitch/actions/modal.js @@ -9,9 +9,10 @@ export function openModal(type, props) { }; }; -export function closeModal(type) { +export function closeModal(type, options = { ignoreFocus: false }) { return { type: MODAL_CLOSE, modalType: type, + ignoreFocus: options.ignoreFocus, }; }; diff --git a/app/javascript/flavours/glitch/components/modal_root.js b/app/javascript/flavours/glitch/components/modal_root.js index 7b5a630e5b..0595f6a0e5 100644 --- a/app/javascript/flavours/glitch/components/modal_root.js +++ b/app/javascript/flavours/glitch/components/modal_root.js @@ -18,6 +18,7 @@ export default class ModalRoot extends React.PureComponent { b: PropTypes.number, }), noEsc: PropTypes.bool, + ignoreFocus: PropTypes.bool, }; activeElement = this.props.children ? document.activeElement : null; @@ -72,7 +73,9 @@ export default class ModalRoot extends React.PureComponent { // immediately selectable, we have to wait for observers to run, as // described in https://github.com/WICG/inert#performance-and-gotchas Promise.resolve().then(() => { - this.activeElement.focus({ preventScroll: true }); + if (!this.props.ignoreFocus) { + this.activeElement.focus({ preventScroll: true }); + } this.activeElement = null; }).catch(console.error); diff --git a/app/javascript/flavours/glitch/features/compose/components/compose_form.js b/app/javascript/flavours/glitch/features/compose/components/compose_form.js index c75906ce72..b03bc34b84 100644 --- a/app/javascript/flavours/glitch/features/compose/components/compose_form.js +++ b/app/javascript/flavours/glitch/features/compose/components/compose_form.js @@ -246,9 +246,14 @@ class ComposeForm extends ImmutablePureComponent { selectionStart = selectionEnd = text.length; } if (textarea) { - textarea.setSelectionRange(selectionStart, selectionEnd); - textarea.focus(); - if (!singleColumn) textarea.scrollIntoView(); + // Because of the wicg-inert polyfill, the activeElement may not be + // immediately selectable, we have to wait for observers to run, as + // described in https://github.com/WICG/inert#performance-and-gotchas + Promise.resolve().then(() => { + textarea.setSelectionRange(selectionStart, selectionEnd); + textarea.focus(); + if (!singleColumn) textarea.scrollIntoView(); + }).catch(console.error); } // Refocuses the textarea after submitting. diff --git a/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js b/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js index e01d277a17..fc0379ce98 100644 --- a/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js +++ b/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js @@ -62,7 +62,7 @@ class Footer extends ImmutablePureComponent { const { router } = this.context; if (onClose) { - onClose(); + onClose(true); } dispatch(replyCompose(status, router.history)); diff --git a/app/javascript/flavours/glitch/features/ui/components/modal_root.js b/app/javascript/flavours/glitch/features/ui/components/modal_root.js index 1e065c1712..a975c4013d 100644 --- a/app/javascript/flavours/glitch/features/ui/components/modal_root.js +++ b/app/javascript/flavours/glitch/features/ui/components/modal_root.js @@ -55,6 +55,7 @@ export default class ModalRoot extends React.PureComponent { type: PropTypes.string, props: PropTypes.object, onClose: PropTypes.func.isRequired, + ignoreFocus: PropTypes.bool, }; state = { @@ -85,7 +86,7 @@ export default class ModalRoot extends React.PureComponent { return ; } - handleClose = () => { + handleClose = (ignoreFocus = false) => { const { onClose } = this.props; let message = null; try { @@ -95,7 +96,7 @@ export default class ModalRoot extends React.PureComponent { // isn't set. // This would be much smoother with react-intl 3+ and `forwardRef`. } - onClose(message); + onClose(message, ignoreFocus); } setModalRef = (c) => { @@ -103,12 +104,12 @@ export default class ModalRoot extends React.PureComponent { } render () { - const { type, props } = this.props; + const { type, props, ignoreFocus } = this.props; const { backgroundColor } = this.state; const visible = !!type; return ( - + {visible && ( {(SpecificComponent) => } diff --git a/app/javascript/flavours/glitch/features/ui/containers/modal_container.js b/app/javascript/flavours/glitch/features/ui/containers/modal_container.js index 039aabd8ae..560c34f014 100644 --- a/app/javascript/flavours/glitch/features/ui/containers/modal_container.js +++ b/app/javascript/flavours/glitch/features/ui/containers/modal_container.js @@ -3,22 +3,23 @@ import { openModal, closeModal } from 'flavours/glitch/actions/modal'; import ModalRoot from '../components/modal_root'; const mapStateToProps = state => ({ - type: state.getIn(['modal', 0, 'modalType'], null), - props: state.getIn(['modal', 0, 'modalProps'], {}), + ignoreFocus: state.getIn(['modal', 'ignoreFocus']), + type: state.getIn(['modal', 'stack', 0, 'modalType'], null), + props: state.getIn(['modal', 'stack', 0, 'modalProps'], {}), }); const mapDispatchToProps = dispatch => ({ - onClose (confirmationMessage) { + onClose (confirmationMessage, ignoreFocus = false) { if (confirmationMessage) { dispatch( openModal('CONFIRM', { message: confirmationMessage.message, confirm: confirmationMessage.confirm, - onConfirm: () => dispatch(closeModal()), + onConfirm: () => dispatch(closeModal(undefined, { ignoreFocus })), }), ); } else { - dispatch(closeModal()); + dispatch(closeModal(undefined, { ignoreFocus })); } }, }); diff --git a/app/javascript/flavours/glitch/reducers/modal.js b/app/javascript/flavours/glitch/reducers/modal.js index ae205c6d54..2ef0aef24f 100644 --- a/app/javascript/flavours/glitch/reducers/modal.js +++ b/app/javascript/flavours/glitch/reducers/modal.js @@ -3,16 +3,36 @@ import { TIMELINE_DELETE } from 'flavours/glitch/actions/timelines'; import { COMPOSE_UPLOAD_CHANGE_SUCCESS } from 'flavours/glitch/actions/compose'; import { Stack as ImmutableStack, Map as ImmutableMap } from 'immutable'; -export default function modal(state = ImmutableStack(), action) { +const initialState = ImmutableMap({ + ignoreFocus: false, + stack: ImmutableStack(), +}); + +const popModal = (state, { modalType, ignoreFocus }) => { + if (modalType === undefined || modalType === state.getIn(['stack', 0, 'modalType'])) { + return state.set('ignoreFocus', !!ignoreFocus).update('stack', stack => stack.shift()); + } else { + return state; + } +}; + +const pushModal = (state, modalType, modalProps) => { + return state.withMutations(map => { + map.set('ignoreFocus', false); + map.update('stack', stack => stack.unshift(ImmutableMap({ modalType, modalProps }))); + }); +}; + +export default function modal(state = initialState, action) { switch(action.type) { case MODAL_OPEN: - return state.unshift(ImmutableMap({ modalType: action.modalType, modalProps: action.modalProps })); + return pushModal(state, action.modalType, action.modalProps); case MODAL_CLOSE: - return (action.modalType === undefined || action.modalType === state.getIn([0, 'modalType'])) ? state.shift() : state; + return popModal(state, action); case COMPOSE_UPLOAD_CHANGE_SUCCESS: - return state.getIn([0, 'modalType']) === 'FOCAL_POINT' ? state.shift() : state; + return popModal(state, { modalType: 'FOCAL_POINT', ignoreFocus: false }); case TIMELINE_DELETE: - return state.filterNot((modal) => modal.get('modalProps').statusId === action.id); + return state.update('stack', stack => stack.filterNot((modal) => modal.get('modalProps').statusId === action.id)); default: return state; } From 4eed5019a23084816931cf9a0f426003aa160256 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 25 Feb 2022 01:20:41 +0100 Subject: [PATCH 16/24] =?UTF-8?q?[Glitch]=20Fix=20media=20modal=20footer's?= =?UTF-8?q?=20=E2=80=9Cexternal=20link=E2=80=9D=20not=20being=20a=20link?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port 255748dff48b80335cfb7af12d1ea67979af09ad to glitch-soc Signed-off-by: Claire --- .../flavours/glitch/components/icon_button.js | 20 +++++++++++++++++-- .../picture_in_picture/components/footer.js | 2 +- .../glitch/styles/components/index.scss | 5 +++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/javascript/flavours/glitch/components/icon_button.js b/app/javascript/flavours/glitch/components/icon_button.js index 58d3568dd2..3999409cdf 100644 --- a/app/javascript/flavours/glitch/components/icon_button.js +++ b/app/javascript/flavours/glitch/components/icon_button.js @@ -30,6 +30,7 @@ export default class IconButton extends React.PureComponent { label: PropTypes.string, counter: PropTypes.number, obfuscateCount: PropTypes.bool, + href: PropTypes.string, }; static defaultProps = { @@ -109,6 +110,7 @@ export default class IconButton extends React.PureComponent { title, counter, obfuscateCount, + href, } = this.props; const { @@ -130,6 +132,21 @@ export default class IconButton extends React.PureComponent { style.width = 'auto'; } + let contents = ( + + + ); + + if (href) { + contents = ( + + {contents} + + ); + } + return ( ); } diff --git a/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js b/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js index fc0379ce98..0408105aee 100644 --- a/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js +++ b/app/javascript/flavours/glitch/features/picture_in_picture/components/footer.js @@ -181,7 +181,7 @@ class Footer extends ImmutablePureComponent { {replyButton} - {withOpenButton && } + {withOpenButton && } ); } diff --git a/app/javascript/flavours/glitch/styles/components/index.scss b/app/javascript/flavours/glitch/styles/components/index.scss index 55abd6e1ea..b6372c096b 100644 --- a/app/javascript/flavours/glitch/styles/components/index.scss +++ b/app/javascript/flavours/glitch/styles/components/index.scss @@ -146,6 +146,11 @@ transition-property: background-color, color; text-decoration: none; + a { + color: inherit; + text-decoration: none; + } + &:hover, &:active, &:focus { From c6df6686af01e774b2b4618e96194bf80db6ecf2 Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 10:30:45 +0100 Subject: [PATCH 17/24] Disable notifications for trending links and trending statuses by default --- app/controllers/settings/preferences_controller.rb | 2 +- app/models/trends.rb | 9 ++++++--- app/models/user.rb | 10 +++++++++- .../settings/preferences/notifications/show.html.haml | 2 ++ config/locales-glitch/simple_form.en.yml | 4 ++++ config/settings.yml | 2 ++ 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb index d05ceb53f6..dfe2ae2e50 100644 --- a/app/controllers/settings/preferences_controller.rb +++ b/app/controllers/settings/preferences_controller.rb @@ -58,7 +58,7 @@ class Settings::PreferencesController < Settings::BaseController :setting_use_pending_items, :setting_trends, :setting_crop_images, - notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag), + notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag trending_link trending_status), interactions: %i(must_be_follower must_be_following must_be_following_dm) ) end diff --git a/app/models/trends.rb b/app/models/trends.rb index f8864e55f5..0be900b048 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -32,10 +32,13 @@ module Trends tags_requiring_review = tags.request_review statuses_requiring_review = statuses.request_review - return if links_requiring_review.empty? && tags_requiring_review.empty? && statuses_requiring_review.empty? - User.staff.includes(:account).find_each do |user| - AdminMailer.new_trends(user.account, links_requiring_review, tags_requiring_review, statuses_requiring_review).deliver_later! if user.allows_trends_review_emails? + links = user.allows_trending_tags_review_emails? ? links_requiring_review : [] + tags = user.allows_trending_links_review_emails? ? tags_requiring_review : [] + statuses = user.allows_trending_statuses_review_emails? ? statuses_requiring_review : [] + next if links.empty? && tags.empty? && statuses.empty? + + AdminMailer.new_trends(user.account, links, tags, statuses).deliver_later! end end diff --git a/app/models/user.rb b/app/models/user.rb index cb03e99a00..77685ad027 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -269,10 +269,18 @@ class User < ApplicationRecord settings.notification_emails['appeal'] end - def allows_trends_review_emails? + def allows_trending_tags_review_emails? settings.notification_emails['trending_tag'] end + def allows_trending_links_review_emails? + settings.notification_emails['trending_link'] + end + + def allows_trending_statuses_review_emails? + settings.notification_emails['trending_status'] + end + def hides_network? @hides_network ||= settings.hide_network end diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml index 223e5d7407..e01cd077ff 100644 --- a/app/views/settings/preferences/notifications/show.html.haml +++ b/app/views/settings/preferences/notifications/show.html.haml @@ -24,6 +24,8 @@ = ff.input :appeal, as: :boolean, wrapper: :with_label = ff.input :pending_account, as: :boolean, wrapper: :with_label = ff.input :trending_tag, as: :boolean, wrapper: :with_label + = ff.input :trending_link, as: :boolean, wrapper: :with_label + = ff.input :trending_status, as: :boolean, wrapper: :with_label .fields-group = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff| diff --git a/config/locales-glitch/simple_form.en.yml b/config/locales-glitch/simple_form.en.yml index 6129435719..c9ef409962 100644 --- a/config/locales-glitch/simple_form.en.yml +++ b/config/locales-glitch/simple_form.en.yml @@ -18,3 +18,7 @@ en: setting_hide_followers_count: Hide your followers count setting_skin: Skin setting_system_emoji_font: Use system's default font for emojis (applies to Glitch flavour only) + notification_emails: + trending_tag: New trending tag requires review + trending_link: New trending link requires review + trending_status: New trending post requires review diff --git a/config/settings.yml b/config/settings.yml index d0946a668e..11709cee4b 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -52,6 +52,8 @@ defaults: &defaults report: true pending_account: true trending_tag: true + trending_link: false + trending_status: false appeal: true interactions: must_be_follower: false From 756f1b6615ac858c958468d1306c39a06db551bb Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 10:45:44 +0100 Subject: [PATCH 18/24] Add option (on by default) to allow toots with content warnings to trend --- app/models/form/admin_settings.rb | 2 ++ app/models/trends/statuses.rb | 2 +- app/views/admin/settings/edit.html.haml | 3 +++ config/locales-glitch/en.yml | 3 +++ config/settings.yml | 1 + 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index 34f14e3124..5627f8a844 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -35,6 +35,7 @@ class Form::AdminSettings show_replies_in_public_timelines trends trendable_by_default + trending_status_cw show_domain_blocks show_domain_blocks_rationale noindex @@ -57,6 +58,7 @@ class Form::AdminSettings show_replies_in_public_timelines trends trendable_by_default + trending_status_cw noindex require_invite_text captcha_enabled diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index e785413ec4..e9c48a06b7 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -93,7 +93,7 @@ class Trends::Statuses < Trends::Base original_status.public_visibility? && original_status.account.discoverable? && !original_status.account.silenced? && - original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply? + (original_status.spoiler_text.blank? || Setting.trending_status_cw) && !original_status.sensitive? && !original_status.reply? end def calculate_scores(statuses, at_time) diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index 49b03a9e35..a287e52ff8 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -86,6 +86,9 @@ .fields-group = f.input :trendable_by_default, as: :boolean, wrapper: :with_label, label: t('admin.settings.trendable_by_default.title'), hint: t('admin.settings.trendable_by_default.desc_html') + .fields-group + = f.input :trending_status_cw, as: :boolean, wrapper: :with_label, label: t('admin.settings.trending_status_cw.title'), hint: t('trending_status_cw.desc_html') + .fields-group = f.input :noindex, as: :boolean, wrapper: :with_label, label: t('admin.settings.default_noindex.title'), hint: t('admin.settings.default_noindex.desc_html') diff --git a/config/locales-glitch/en.yml b/config/locales-glitch/en.yml index c382ee9ed2..7fd0683c99 100644 --- a/config/locales-glitch/en.yml +++ b/config/locales-glitch/en.yml @@ -20,6 +20,9 @@ en: show_replies_in_public_timelines: desc_html: In addition to public self-replies (threads), show public replies in local and public timelines. title: Show replies in public timelines + trending_status_cw: + desc_html: When trending posts are enabled, allow posts with Content Warnings to be eligible. Changes to this setting are not retroactive. + title: Allow posts with Content Warnings to trend auth: captcha_confirmation: hint_html: Just one more step! To confirm your account, this server requires you to solve a CAPTCHA. You can contact the server administrator if you have questions or need assistance with confirming your account. diff --git a/config/settings.yml b/config/settings.yml index 11709cee4b..1f366d7d11 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -41,6 +41,7 @@ defaults: &defaults use_pending_items: false trends: true trendable_by_default: false + trending_status_cw: true crop_images: true notification_emails: follow: false From 3d60708508c6bfc5b6635aff0482d640a5f318ca Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 16:28:28 +0100 Subject: [PATCH 19/24] Fix crash in EmailDomainBlockRefreshScheduler (#17649) --- app/workers/scheduler/email_domain_block_refresh_scheduler.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/workers/scheduler/email_domain_block_refresh_scheduler.rb b/app/workers/scheduler/email_domain_block_refresh_scheduler.rb index c67be68438..e0ad89866b 100644 --- a/app/workers/scheduler/email_domain_block_refresh_scheduler.rb +++ b/app/workers/scheduler/email_domain_block_refresh_scheduler.rb @@ -15,7 +15,8 @@ class Scheduler::EmailDomainBlockRefreshScheduler if ip?(email_domain_block.domain) [email_domain_block.domain] else - dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::A).to_a + dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::AAAA).to_a.map { |resource| resource.address.to_s } + resources = dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::A).to_a + dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::AAAA).to_a + resources.map { |resource| resource.address.to_s } end end From 6aef76b5cde2315135d53215d13a9b2ec0a1adaa Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 17:26:28 +0100 Subject: [PATCH 20/24] Fix error when a MX is shared across blocked domains (#17650) --- app/controllers/admin/email_domain_blocks_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/admin/email_domain_blocks_controller.rb b/app/controllers/admin/email_domain_blocks_controller.rb index 33ee079f36..a4bbbba5ba 100644 --- a/app/controllers/admin/email_domain_blocks_controller.rb +++ b/app/controllers/admin/email_domain_blocks_controller.rb @@ -38,6 +38,8 @@ module Admin log_action :create, @email_domain_block (@email_domain_block.other_domains || []).uniq.each do |domain| + next if EmailDomainBlock.where(domain: domain).exists? + other_email_domain_block = EmailDomainBlock.create!(domain: domain, parent: @email_domain_block) log_action :create, other_email_domain_block end From 48caeb9d659abd58ec7a9dc04f7365b35e314b74 Mon Sep 17 00:00:00 2001 From: LinAGKar Date: Sat, 26 Feb 2022 17:27:11 +0100 Subject: [PATCH 21/24] Also compress SVG and ICO images in nginx (#17651) --- dist/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/nginx.conf b/dist/nginx.conf index 27ca868ab0..7e03343680 100644 --- a/dist/nginx.conf +++ b/dist/nginx.conf @@ -50,7 +50,7 @@ server { gzip_comp_level 6; gzip_buffers 16 8k; gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/x-icon; add_header Strict-Transport-Security "max-age=31536000" always; From 0dc57ab6ed67657e0a77e08bcd99c7b809fe5e42 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 26 Feb 2022 17:51:59 +0100 Subject: [PATCH 22/24] Fix status updates not being forwarded like deletes through ActivityPub (#17648) Fix #17521 --- app/lib/activitypub/activity/delete.rb | 40 ++-------------- app/lib/activitypub/activity/update.rb | 11 +++-- app/lib/activitypub/forwarder.rb | 65 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 40 deletions(-) create mode 100644 app/lib/activitypub/forwarder.rb diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 801647cf74..f5ef863f37 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -37,50 +37,16 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity return if @status.nil? - forward! if @json['signature'].present? && @status.distributable? + forwarder.forward! if forwarder.forwardable? delete_now! end end - def rebloggers_ids - return @rebloggers_ids if defined?(@rebloggers_ids) - @rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id) - end - - def inboxes_for_reblogs - Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes - end - - def replied_to_status - return @replied_to_status if defined?(@replied_to_status) - @replied_to_status = @status.thread - end - - def reply_to_local? - !replied_to_status.nil? && replied_to_status.account.local? - end - - def inboxes_for_reply - replied_to_status.account.followers.inboxes - end - - def forward! - inboxes = inboxes_for_reblogs - inboxes += inboxes_for_reply if reply_to_local? - inboxes -= [@account.preferred_inbox_url] - - sender_id = reply_to_local? ? replied_to_status.account_id : rebloggers_ids.first - - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes.uniq) do |inbox_url| - [payload, sender_id, inbox_url] - end + def forwarder + @forwarder ||= ActivityPub::Forwarder.new(@account, @json, @status) end def delete_now! RemoveStatusService.new.call(@status, redraft: false) end - - def payload - @payload ||= Oj.dump(@json) - end end diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index f04ad321b2..0bfead55b2 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -22,10 +22,15 @@ class ActivityPub::Activity::Update < ActivityPub::Activity def update_status return reject_payload! if invalid_origin?(@object['id']) - status = Status.find_by(uri: object_uri, account_id: @account.id) + @status = Status.find_by(uri: object_uri, account_id: @account.id) - return if status.nil? + return if @status.nil? - ActivityPub::ProcessStatusUpdateService.new.call(status, @object) + forwarder.forward! if forwarder.forwardable? + ActivityPub::ProcessStatusUpdateService.new.call(@status, @object) + end + + def forwarder + @forwarder ||= ActivityPub::Forwarder.new(@account, @json, @status) end end diff --git a/app/lib/activitypub/forwarder.rb b/app/lib/activitypub/forwarder.rb new file mode 100644 index 0000000000..4206b9d822 --- /dev/null +++ b/app/lib/activitypub/forwarder.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class ActivityPub::Forwarder + def initialize(account, original_json, status) + @json = original_json + @account = account + @status = status + end + + def forwardable? + @json['signature'].present? && @status.distributable? + end + + def forward! + ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| + [payload, signature_account_id, inbox_url] + end + end + + private + + def payload + @payload ||= Oj.dump(@json) + end + + def reblogged_by_account_ids + @reblogged_by_account_ids ||= @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id) + end + + def signature_account_id + @signature_account_id ||= begin + if in_reply_to_local? + in_reply_to.account_id + else + reblogged_by_account_ids.first + end + end + end + + def inboxes + @inboxes ||= begin + arr = inboxes_for_followers_of_reblogged_by_accounts + arr += inboxes_for_followers_of_replied_to_account if in_reply_to_local? + arr -= [@account.preferred_inbox_url] + arr.uniq! + arr + end + end + + def inboxes_for_followers_of_reblogged_by_accounts + Account.where(id: ::Follow.where(target_account_id: reblogged_by_account_ids).select(:account_id)).inboxes + end + + def inboxes_for_followers_of_replied_to_account + in_reply_to.account.followers.inboxes + end + + def in_reply_to + @status.thread + end + + def in_reply_to_local? + @status.thread&.account&.local? + end +end From 57814a98a9c8e4b106d44a31e36561f585f73bac Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 21:14:12 +0100 Subject: [PATCH 23/24] Fix remote reports with comments revealing remote reporter (#17652) * Display username rather than display name in report comment For consistency with report notes and appeals * Fix remote reports with comments revealing remote reporter * Display instance name in placeholder * Make instance name in report comment a link to the federation admin page * Normalize i18n file --- app/javascript/styles/mastodon/admin.scss | 16 ++++++++++------ app/views/admin/reports/show.html.haml | 19 ++++++++++++++++--- config/locales/en.yml | 1 + 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss index ee7b26d076..2e212eca52 100644 --- a/app/javascript/styles/mastodon/admin.scss +++ b/app/javascript/styles/mastodon/admin.scss @@ -1367,16 +1367,20 @@ a.sparkline { line-height: 20px; margin-bottom: 4px; - .username a { + .username { color: $primary-text-color; font-weight: 500; - text-decoration: none; margin-right: 5px; - &:hover, - &:focus, - &:active { - text-decoration: underline; + a { + color: inherit; + text-decoration: none; + + &:hover, + &:focus, + &:active { + text-decoration: underline; + } } } diff --git a/app/views/admin/reports/show.html.haml b/app/views/admin/reports/show.html.haml index e53c180e50..25b751335f 100644 --- a/app/views/admin/reports/show.html.haml +++ b/app/views/admin/reports/show.html.haml @@ -122,15 +122,28 @@ = react_admin_component :report_reason_selector, id: @report.id, category: @report.category, rule_ids: @report.rule_ids&.map(&:to_s), disabled: @report.action_taken? - if @report.comment.present? - %p= t('admin.reports.comment_description_html', name: content_tag(:strong, @report.account.username, class: 'username')) + - if @report.account.instance_actor? + %p= t('admin.reports.comment_description_html', name: content_tag(:strong, site_hostname, class: 'username')) + - elsif @report.account.local? + %p= t('admin.reports.comment_description_html', name: content_tag(:strong, @report.account.username, class: 'username')) + - else + %p= t('admin.reports.comment_description_html', name: t('admin.reports.remote_user_placeholder', instance: @report.account.domain)) .report-notes .report-notes__item - = image_tag @report.account.avatar.url, class: 'report-notes__item__avatar' + - if @report.account.local? && !@report.account.instance_actor? + = image_tag @report.account.avatar.url, class: 'report-notes__item__avatar' + - else + = image_tag(full_asset_url('avatars/original/missing.png', skip_pipeline: true), class: 'report-notes__item__avatar') .report-notes__item__header %span.username - = link_to display_name(@report.account), admin_account_path(@report.account_id) + - if @report.account.instance_actor? + = site_hostname + - elsif @report.account.local? + = link_to @report.account.username, admin_account_path(@report.account_id) + - else + = link_to @report.account.domain, admin_instance_path(@report.account.domain) %time{ datetime: @report.created_at.iso8601, title: l(@report.created_at) } - if @report.created_at.today? = t('admin.report_notes.today_at', time: l(@report.created_at, format: :time)) diff --git a/config/locales/en.yml b/config/locales/en.yml index 60c2915407..536d1dbf65 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -617,6 +617,7 @@ en: title: Notes notes_description_html: View and leave notes to other moderators and your future self quick_actions_description_html: 'Take a quick action or scroll down to see reported content:' + remote_user_placeholder: the remote user from %{instance} reopen: Reopen report report: 'Report #%{id}' reported_account: Reported account From 0a5a703a56a1f93fda84d537bc09bfbe45210494 Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 26 Feb 2022 21:14:12 +0100 Subject: [PATCH 24/24] [Glitch] Fix remote reports with comments revealing remote reporter Port SCSS changes from 57814a98a9c8e4b106d44a31e36561f585f73bac to glitch-soc Signed-off-by: Claire --- app/javascript/flavours/glitch/styles/admin.scss | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/javascript/flavours/glitch/styles/admin.scss b/app/javascript/flavours/glitch/styles/admin.scss index a1b99636c7..fa47d19d7c 100644 --- a/app/javascript/flavours/glitch/styles/admin.scss +++ b/app/javascript/flavours/glitch/styles/admin.scss @@ -1383,16 +1383,20 @@ a.sparkline { line-height: 20px; margin-bottom: 4px; - .username a { + .username { color: $primary-text-color; font-weight: 500; - text-decoration: none; margin-right: 5px; - &:hover, - &:focus, - &:active { - text-decoration: underline; + a { + color: inherit; + text-decoration: none; + + &:hover, + &:focus, + &:active { + text-decoration: underline; + } } }