Add trending links (#16917)
* Add trending links * Add overriding specific links trendability * Add link type to preview cards and only trend articles Change trends review notifications from being sent every 5 minutes to being sent every 2 hours Change threshold from 5 unique accounts to 15 unique accounts * Fix testslocal
parent
46e62fc4b3
commit
6e50134a42
97 changed files with 2064 additions and 715 deletions
@ -0,0 +1,41 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController |
||||
def index |
||||
authorize :preview_card_provider, :index? |
||||
|
||||
@preview_card_providers = filtered_preview_card_providers.page(params[:page]) |
||||
@form = Form::PreviewCardProviderBatch.new |
||||
end |
||||
|
||||
def batch |
||||
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) |
||||
@form.save |
||||
rescue ActionController::ParameterMissing |
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected') |
||||
ensure |
||||
redirect_to admin_trends_links_preview_card_providers_path(filter_params) |
||||
end |
||||
|
||||
private |
||||
|
||||
def filtered_preview_card_providers |
||||
PreviewCardProviderFilter.new(filter_params).results |
||||
end |
||||
|
||||
def filter_params |
||||
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS) |
||||
end |
||||
|
||||
def form_preview_card_provider_batch_params |
||||
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) |
||||
end |
||||
|
||||
def action_from_button |
||||
if params[:approve] |
||||
'approve' |
||||
elsif params[:reject] |
||||
'reject' |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,45 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Trends::LinksController < Admin::BaseController |
||||
def index |
||||
authorize :preview_card, :index? |
||||
|
||||
@preview_cards = filtered_preview_cards.page(params[:page]) |
||||
@form = Form::PreviewCardBatch.new |
||||
end |
||||
|
||||
def batch |
||||
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) |
||||
@form.save |
||||
rescue ActionController::ParameterMissing |
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected') |
||||
ensure |
||||
redirect_to admin_trends_links_path(filter_params) |
||||
end |
||||
|
||||
private |
||||
|
||||
def filtered_preview_cards |
||||
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results |
||||
end |
||||
|
||||
def filter_params |
||||
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS) |
||||
end |
||||
|
||||
def form_preview_card_batch_params |
||||
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: []) |
||||
end |
||||
|
||||
def action_from_button |
||||
if params[:approve] |
||||
'approve' |
||||
elsif params[:approve_all] |
||||
'approve_all' |
||||
elsif params[:reject] |
||||
'reject' |
||||
elsif params[:reject_all] |
||||
'reject_all' |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,41 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Trends::TagsController < Admin::BaseController |
||||
def index |
||||
authorize :tag, :index? |
||||
|
||||
@tags = filtered_tags.page(params[:page]) |
||||
@form = Form::TagBatch.new |
||||
end |
||||
|
||||
def batch |
||||
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) |
||||
@form.save |
||||
rescue ActionController::ParameterMissing |
||||
flash[:alert] = I18n.t('admin.accounts.no_account_selected') |
||||
ensure |
||||
redirect_to admin_trends_tags_path(filter_params) |
||||
end |
||||
|
||||
private |
||||
|
||||
def filtered_tags |
||||
TagFilter.new(filter_params).results |
||||
end |
||||
|
||||
def filter_params |
||||
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) |
||||
end |
||||
|
||||
def form_tag_batch_params |
||||
params.require(:form_tag_batch).permit(:action, tag_ids: []) |
||||
end |
||||
|
||||
def action_from_button |
||||
if params[:approve] |
||||
'approve' |
||||
elsif params[:reject] |
||||
'reject' |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,16 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Api::V1::Admin::Trends::TagsController < Api::BaseController |
||||
before_action :require_staff! |
||||
before_action :set_tags |
||||
|
||||
def index |
||||
render json: @tags, each_serializer: REST::Admin::TagSerializer |
||||
end |
||||
|
||||
private |
||||
|
||||
def set_tags |
||||
@tags = Trends.tags.get(false, limit_param(10)) |
||||
end |
||||
end |
@ -1,16 +0,0 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Api::V1::Admin::TrendsController < Api::BaseController |
||||
before_action :require_staff! |
||||
before_action :set_trends |
||||
|
||||
def index |
||||
render json: @trends, each_serializer: REST::Admin::TagSerializer |
||||
end |
||||
|
||||
private |
||||
|
||||
def set_trends |
||||
@trends = TrendingTags.get(10, filtered: false) |
||||
end |
||||
end |
@ -0,0 +1,21 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Api::V1::Trends::LinksController < Api::BaseController |
||||
before_action :set_links |
||||
|
||||
def index |
||||
render json: @links, each_serializer: REST::Trends::LinkSerializer |
||||
end |
||||
|
||||
private |
||||
|
||||
def set_links |
||||
@links = begin |
||||
if Setting.trends |
||||
Trends.links.get(true, limit_param(10)) |
||||
else |
||||
[] |
||||
end |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,21 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Api::V1::Trends::TagsController < Api::BaseController |
||||
before_action :set_tags |
||||
|
||||
def index |
||||
render json: @tags, each_serializer: REST::TagSerializer |
||||
end |
||||
|
||||
private |
||||
|
||||
def set_tags |
||||
@tags = begin |
||||
if Setting.trends |
||||
Trends.tags.get(true, limit_param(10)) |
||||
else |
||||
[] |
||||
end |
||||
end |
||||
end |
||||
end |
@ -1,15 +0,0 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Api::V1::TrendsController < Api::BaseController |
||||
before_action :set_tags |
||||
|
||||
def index |
||||
render json: @tags, each_serializer: REST::TagSerializer |
||||
end |
||||
|
||||
private |
||||
|
||||
def set_tags |
||||
@tags = TrendingTags.get(limit_param(10)) |
||||
end |
||||
end |
@ -0,0 +1,36 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension |
||||
include LanguagesHelper |
||||
|
||||
def self.with_params? |
||||
true |
||||
end |
||||
|
||||
def key |
||||
'tag_languages' |
||||
end |
||||
|
||||
def data |
||||
sql = <<-SQL.squish |
||||
SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value |
||||
FROM statuses |
||||
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id |
||||
WHERE statuses_tags.tag_id = $1 |
||||
AND statuses.id BETWEEN $2 AND $3 |
||||
GROUP BY COALESCE(statuses.language, 'und') |
||||
ORDER BY count(*) DESC |
||||
LIMIT $4 |
||||
SQL |
||||
|
||||
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) |
||||
|
||||
rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } } |
||||
end |
||||
|
||||
private |
||||
|
||||
def params |
||||
@params.permit(:id) |
||||
end |
||||
end |
@ -0,0 +1,35 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension |
||||
def self.with_params? |
||||
true |
||||
end |
||||
|
||||
def key |
||||
'tag_servers' |
||||
end |
||||
|
||||
def data |
||||
sql = <<-SQL.squish |
||||
SELECT accounts.domain, count(*) AS value |
||||
FROM statuses |
||||
INNER JOIN accounts ON accounts.id = statuses.account_id |
||||
INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id |
||||
WHERE statuses_tags.tag_id = $1 |
||||
AND statuses.id BETWEEN $2 AND $3 |
||||
GROUP BY accounts.domain |
||||
ORDER BY count(*) DESC |
||||
LIMIT $4 |
||||
SQL |
||||
|
||||
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) |
||||
|
||||
rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } |
||||
end |
||||
|
||||
private |
||||
|
||||
def params |
||||
@params.permit(:id) |
||||
end |
||||
end |
@ -0,0 +1,41 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure |
||||
def self.with_params? |
||||
true |
||||
end |
||||
|
||||
def key |
||||
'tag_accounts' |
||||
end |
||||
|
||||
def total |
||||
tag.history.aggregate(time_period).accounts |
||||
end |
||||
|
||||
def previous_total |
||||
tag.history.aggregate(previous_time_period).accounts |
||||
end |
||||
|
||||
def data |
||||
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } |
||||
end |
||||
|
||||
protected |
||||
|
||||
def tag |
||||
@tag ||= Tag.find(params[:id]) |
||||
end |
||||
|
||||
def time_period |
||||
(@start_at.to_date..@end_at.to_date) |
||||
end |
||||
|
||||
def previous_time_period |
||||
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) |
||||
end |
||||
|
||||
def params |
||||
@params.permit(:id) |
||||
end |
||||
end |
@ -0,0 +1,47 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure |
||||
def self.with_params? |
||||
true |
||||
end |
||||
|
||||
def key |
||||
'tag_servers' |
||||
end |
||||
|
||||
def total |
||||
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') |
||||
end |
||||
|
||||
def previous_total |
||||
tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') |
||||
end |
||||
|
||||
def data |
||||
sql = <<-SQL.squish |
||||
SELECT axis.*, ( |
||||
SELECT count(*) AS value |
||||
FROM statuses |
||||
WHERE statuses.id BETWEEN $1 AND $2 |
||||
AND date_trunc('day', statuses.created_at)::date = axis.day |
||||
) |
||||
FROM ( |
||||
SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day |
||||
) as axis |
||||
SQL |
||||
|
||||
rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]]) |
||||
|
||||
rows.map { |row| { date: row['day'], value: row['value'].to_s } } |
||||
end |
||||
|
||||
protected |
||||
|
||||
def tag |
||||
@tag ||= Tag.find(params[:id]) |
||||
end |
||||
|
||||
def params |
||||
@params.permit(:id) |
||||
end |
||||
end |
@ -0,0 +1,41 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure |
||||
def self.with_params? |
||||
true |
||||
end |
||||
|
||||
def key |
||||
'tag_uses' |
||||
end |
||||
|
||||
def total |
||||
tag.history.aggregate(time_period).uses |
||||
end |
||||
|
||||
def previous_total |
||||
tag.history.aggregate(previous_time_period).uses |
||||
end |
||||
|
||||
def data |
||||
time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } |
||||
end |
||||
|
||||
protected |
||||
|
||||
def tag |
||||
@tag ||= Tag.find(params[:id]) |
||||
end |
||||
|
||||
def time_period |
||||
(@start_at.to_date..@end_at.to_date) |
||||
end |
||||
|
||||
def previous_time_period |
||||
((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) |
||||
end |
||||
|
||||
def params |
||||
@params.permit(:id) |
||||
end |
||||
end |
@ -0,0 +1,65 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Form::PreviewCardBatch |
||||
include ActiveModel::Model |
||||
include Authorization |
||||
|
||||
attr_accessor :preview_card_ids, :action, :current_account, :precision |
||||
|
||||
def save |
||||
case action |
||||
when 'approve' |
||||
approve! |
||||
when 'approve_all' |
||||
approve_all! |
||||
when 'reject' |
||||
reject! |
||||
when 'reject_all' |
||||
reject_all! |
||||
end |
||||
end |
||||
|
||||
private |
||||
|
||||
def preview_cards |
||||
@preview_cards ||= PreviewCard.where(id: preview_card_ids) |
||||
end |
||||
|
||||
def preview_card_providers |
||||
@preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) } |
||||
end |
||||
|
||||
def approve! |
||||
preview_cards.each { |preview_card| authorize(preview_card, :update?) } |
||||
preview_cards.update_all(trendable: true) |
||||
end |
||||
|
||||
def approve_all! |
||||
preview_card_providers.each do |provider| |
||||
authorize(provider, :update?) |
||||
provider.update(trendable: true, reviewed_at: action_time) |
||||
end |
||||
|
||||
# Reset any individual overrides |
||||
preview_cards.update_all(trendable: nil) |
||||
end |
||||
|
||||
def reject! |
||||
preview_cards.each { |preview_card| authorize(preview_card, :update?) } |
||||
preview_cards.update_all(trendable: false) |
||||
end |
||||
|
||||
def reject_all! |
||||
preview_card_providers.each do |provider| |
||||
authorize(provider, :update?) |
||||
provider.update(trendable: false, reviewed_at: action_time) |
||||
end |
||||
|
||||
# Reset any individual overrides |
||||
preview_cards.update_all(trendable: nil) |
||||
end |
||||
|
||||
def action_time |
||||
@action_time ||= Time.now.utc |
||||
end |
||||
end |
@ -0,0 +1,33 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Form::PreviewCardProviderBatch |
||||
include ActiveModel::Model |
||||
include Authorization |
||||
|
||||
attr_accessor :preview_card_provider_ids, :action, :current_account |
||||
|
||||
def save |
||||
case action |
||||
when 'approve' |
||||
approve! |
||||
when 'reject' |
||||
reject! |
||||
end |
||||
end |
||||
|
||||
private |
||||
|
||||
def preview_card_providers |
||||
PreviewCardProvider.where(id: preview_card_provider_ids) |
||||
end |
||||
|
||||
def approve! |
||||
preview_card_providers.each { |provider| authorize(provider, :update?) } |
||||
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) |
||||
end |
||||
|
||||
def reject! |
||||
preview_card_providers.each { |provider| authorize(provider, :update?) } |
||||
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) |
||||
end |
||||
end |
@ -0,0 +1,53 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class PreviewCardFilter |
||||
KEYS = %i( |
||||
trending |
||||
).freeze |
||||
|
||||
attr_reader :params |
||||
|
||||
def initialize(params) |
||||
@params = params |
||||
end |
||||
|
||||
def results |
||||
scope = PreviewCard.unscoped |
||||
|
||||
params.each do |key, value| |
||||
next if key.to_s == 'page' |
||||
|
||||
scope.merge!(scope_for(key, value.to_s.strip)) if value.present? |
||||
end |
||||
|
||||
scope |
||||
end |
||||
|
||||
private |
||||
|
||||
def scope_for(key, value) |
||||
case key.to_s |
||||
when 'trending' |
||||
trending_scope(value) |
||||
else |
||||
raise "Unknown filter: #{key}" |
||||
end |
||||
end |
||||
|
||||
def trending_scope(value) |
||||
ids = begin |
||||
case value.to_s |
||||
when 'allowed' |
||||
Trends.links.currently_trending_ids(true, -1) |
||||
else |
||||
Trends.links.currently_trending_ids(false, -1) |
||||
end |
||||
end |
||||
|
||||
if ids.empty? |
||||
PreviewCard.none |
||||
else |
||||
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,57 @@ |
||||
# frozen_string_literal: true |
||||
# == Schema Information |
||||
# |
||||
# Table name: preview_card_providers |
||||
# |
||||
# id :bigint(8) not null, primary key |
||||
# domain :string default(""), not null |
||||
# icon_file_name :string |
||||
# icon_content_type :string |
||||
# icon_file_size :bigint(8) |
||||
# icon_updated_at :datetime |
||||
# trendable :boolean |
||||
# reviewed_at :datetime |
||||
# requested_review_at :datetime |
||||
# created_at :datetime not null |
||||
# updated_at :datetime not null |
||||
# |
||||
|
||||
class PreviewCardProvider < ApplicationRecord |
||||
include DomainNormalizable |
||||
include Attachmentable |
||||
|
||||
ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze |
||||
LIMIT = 1.megabyte |
||||
|
||||
validates :domain, presence: true, uniqueness: true, domain: true |
||||
|
||||
has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false |
||||
validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT } |
||||
remotable_attachment :icon, LIMIT |
||||
|
||||
scope :trendable, -> { where(trendable: true) } |
||||
scope :not_trendable, -> { where(trendable: false) } |
||||
scope :reviewed, -> { where.not(reviewed_at: nil) } |
||||
scope :pending_review, -> { where(reviewed_at: nil) } |
||||
|
||||
def requires_review? |
||||
reviewed_at.nil? |
||||
end |
||||
|
||||
def reviewed? |
||||
reviewed_at.present? |
||||
end |
||||
|
||||
def requested_review? |
||||
requested_review_at.present? |
||||
end |
||||
|
||||
def requires_review_notification? |
||||
requires_review? && !requested_review? |
||||
end |
||||
|
||||
def self.matching_domain(domain) |
||||
segments = domain.split('.') |
||||
where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first |
||||
end |
||||
end |
@ -0,0 +1,49 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class PreviewCardProviderFilter |
||||
KEYS = %i( |
||||
status |
||||
).freeze |
||||
|
||||
attr_reader :params |
||||
|
||||
def initialize(params) |
||||
@params = params |
||||
end |
||||
|
||||
def results |
||||
scope = PreviewCardProvider.unscoped |
||||
|
||||
params.each do |key, value| |
||||
next if key.to_s == 'page' |
||||
|
||||
scope.merge!(scope_for(key, value.to_s.strip)) if value.present? |
||||
end |
||||
|
||||
scope.order(domain: :asc) |
||||
end |
||||
|
||||
private |
||||
|
||||
def scope_for(key, value) |
||||
case key.to_s |
||||
when 'status' |
||||
status_scope(value) |
||||
else |
||||
raise "Unknown filter: #{key}" |
||||
end |
||||
end |
||||
|
||||
def status_scope(value) |
||||
case value.to_s |
||||
when 'approved' |
||||
PreviewCardProvider.trendable |
||||
when 'rejected' |
||||
PreviewCardProvider.not_trendable |
||||
when 'pending_review' |
||||
PreviewCardProvider.pending_review |
||||
else |
||||
raise "Unknown status: #{value}" |
||||
end |
||||
end |
||||
end |
@ -1,128 +0,0 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class TrendingTags |
||||
KEY = 'trending_tags' |
||||
EXPIRE_HISTORY_AFTER = 7.days.seconds |
||||
EXPIRE_TRENDS_AFTER = 1.day.seconds |
||||
THRESHOLD = 5 |
||||
LIMIT = 10 |
||||
REVIEW_THRESHOLD = 3 |
||||
MAX_SCORE_COOLDOWN = 2.days.freeze |
||||
MAX_SCORE_HALFLIFE = 2.hours.freeze |
||||
|
||||
class << self |
||||
include Redisable |
||||
|
||||
def record_use!(tag, account, status: nil, at_time: Time.now.utc) |
||||
return unless tag.usable? && !account.silenced? |
||||
|
||||
# Even if a tag is not allowed to trend, we still need to |
||||
# record the stats since they can be displayed in other places |
||||
increment_historical_use!(tag.id, at_time) |
||||
increment_unique_use!(tag.id, account.id, at_time) |
||||
increment_use!(tag.id, at_time) |
||||
|
||||
# Only update when the tag was last used once every 12 hours |
||||
# and only if a status is given (lets use ignore reblogs) |
||||
tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago)) |
||||
end |
||||
|
||||
def update!(at_time = Time.now.utc) |
||||
tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) |
||||
tags = Tag.trendable.where(id: tag_ids.uniq) |
||||
|
||||
# First pass to calculate scores and update the set |
||||
|
||||
tags.each do |tag| |
||||
expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f |
||||
expected = 1.0 if expected.zero? |
||||
observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f |
||||
max_time = tag.max_score_at |
||||
max_score = tag.max_score |
||||
max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) |
||||
|
||||
score = begin |
||||
if expected > observed || observed < THRESHOLD |
||||
0 |
||||
else |
||||
((observed - expected)**2) / expected |
||||
end |
||||
end |
||||
|
||||
if score > max_score |
||||
max_score = score |
||||
max_time = at_time |
||||
|
||||
# Not interested in triggering any callbacks for this |
||||
tag.update_columns(max_score: max_score, max_score_at: max_time) |
||||
end |
||||
|
||||
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) |
||||
|
||||
if decaying_score.zero? |
||||
redis.zrem(KEY, tag.id) |
||||
else |
||||
redis.zadd(KEY, decaying_score, tag.id) |
||||
end |
||||
end |
||||
|
||||
users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) |
||||
|
||||
# Second pass to notify about previously unreviewed trends |
||||
|
||||
tags.each do |tag| |
||||
current_rank = redis.zrevrank(KEY, tag.id) |
||||
needs_review_notification = tag.requires_review? && !tag.requested_review? |
||||
rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD |
||||
|
||||
next unless !tag.trendable? && rank_passes_threshold && needs_review_notification |
||||
|
||||
tag.touch(:requested_review_at) |
||||
|
||||
users_for_review.each do |user| |
||||
AdminMailer.new_trending_tag(user.account, tag).deliver_later! |
||||
end |
||||
end |
||||
|
||||
# Trim older items |
||||
|
||||
redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) |
||||
redis.zremrangebyscore(KEY, '(0.3', '-inf') |
||||
end |
||||
|
||||
def get(limit, filtered: true) |
||||
tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) |
||||
|
||||
tags = Tag.where(id: tag_ids) |
||||
tags = tags.trendable if filtered |
||||
tags = tags.index_by(&:id) |
||||
|
||||
tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit) |
||||
end |
||||
|
||||
def trending?(tag) |
||||
rank = redis.zrevrank(KEY, tag.id) |
||||
rank.present? && rank < LIMIT |
||||
end |
||||
|
||||
private |
||||
|
||||
def increment_historical_use!(tag_id, at_time) |
||||
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" |
||||
redis.incrby(key, 1) |
||||
redis.expire(key, EXPIRE_HISTORY_AFTER) |
||||
end |
||||
|
||||
def increment_unique_use!(tag_id, account_id, at_time) |
||||
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" |
||||
redis.pfadd(key, account_id) |
||||
redis.expire(key, EXPIRE_HISTORY_AFTER) |
||||
end |
||||
|
||||
def increment_use!(tag_id, at_time) |
||||
key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" |
||||
redis.sadd(key, tag_id) |
||||
redis.expire(key, EXPIRE_HISTORY_AFTER) |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,27 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
module Trends |
||||
def self.table_name_prefix |
||||
'trends_' |
||||
end |
||||
|
||||
def self.links |
||||
@links ||= Trends::Links.new |
||||
end |
||||
|
||||
def self.tags |
||||
@tags ||= Trends::Tags.new |
||||
end |
||||
|
||||
def self.refresh! |
||||
[links, tags].each(&:refresh) |
||||
end |
||||
|
||||
def self.request_review! |
||||
[links, tags].each(&:request_review) if enabled? |
||||
end |
||||
|
||||
def self.enabled? |
||||
Setting.trends |
||||
end |
||||
end |
@ -0,0 +1,80 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Trends::Base |
||||
include Redisable |
||||
|
||||
class_attribute :default_options |
||||
|
||||
attr_reader :options |
||||
|
||||
# @param [Hash] options |
||||
# @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score |
||||
# @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review |
||||
# @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from |
||||
# @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays |
||||
def initialize(options = {}) |
||||
@options = self.class.default_options.merge(options) |
||||
end |
||||
|
||||
def register(_status) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def add(*) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def refresh(*) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def request_review |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def get(*) |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def score(id) |
||||
redis.zscore("#{key_prefix}:all", id) || 0 |
||||
end |
||||
|
||||
def rank(id) |
||||
redis.zrevrank("#{key_prefix}:allowed", id) |
||||
end |
||||
|
||||
def currently_trending_ids(allowed, limit) |
||||
redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i) |
||||
end |
||||
|
||||
protected |
||||
|
||||
def key_prefix |
||||
raise NotImplementedError |
||||
end |
||||
|
||||
def recently_used_ids(at_time = Time.now.utc) |
||||
redis.smembers(used_key(at_time)).map(&:to_i) |
||||
end |
||||
|
||||
def record_used_id(id, at_time = Time.now.utc) |
||||
redis.sadd(used_key(at_time), id) |
||||
redis.expire(used_key(at_time), 1.day.seconds) |
||||
end |
||||
|
||||
def trim_older_items |
||||
redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1') |
||||
redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1') |
||||
end |
||||
|
||||
def score_at_rank(rank) |
||||
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 |
||||
end |
||||
|
||||
private |
||||
|
||||
def used_key(at_time) |
||||
"#{key_prefix}:used:#{at_time.beginning_of_day.to_i}" |
||||
end |
||||
end |
@ -0,0 +1,98 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Trends::History |
||||
include Enumerable |
||||
|
||||
class Aggregate |
||||
include Redisable |
||||
|
||||
def initialize(prefix, id, date_range) |
||||
@days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) } |
||||
end |
||||
|
||||
def uses |
||||
redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum |
||||
end |
||||
|
||||
def accounts |
||||
redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) |
||||
end |
||||
end |
||||
|
||||
class Day |
||||
include Redisable |
||||
|
||||
EXPIRE_AFTER = 14.days.seconds |
||||
|
||||
def initialize(prefix, id, day) |
||||
@prefix = prefix |
||||
@id = id |
||||
@day = day.beginning_of_day |
||||
end |
||||
|
||||
attr_reader :day |
||||
|
||||
def accounts |
||||
redis.pfcount(key_for(:accounts)) |
||||
end |
||||
|
||||
def uses |
||||
redis.get(key_for(:uses))&.to_i || 0 |
||||
end |
||||
|
||||
def add(account_id) |
||||
redis.pipelined do |
||||
redis.incrby(key_for(:uses), 1) |
||||
redis.pfadd(key_for(:accounts), account_id) |
||||
redis.expire(key_for(:uses), EXPIRE_AFTER) |
||||
redis.expire(key_for(:accounts), EXPIRE_AFTER) |
||||
end |
||||
end |
||||
|
||||
def as_json |
||||
{ day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s } |
||||
end |
||||
|
||||
def key_for(suffix) |
||||
case suffix |
||||
when :accounts |
||||
"#{key_prefix}:#{suffix}" |
||||
when :uses |
||||
key_prefix |
||||
end |
||||
end |
||||
|
||||
def key_prefix |
||||
"activity:#{@prefix}:#{@id}:#{day.to_i}" |
||||
end |
||||
end |
||||
|
||||
def initialize(prefix, id) |
||||
@prefix = prefix |
||||
@id = id |
||||
end |
||||
|
||||
def get(date) |
||||
Day.new(@prefix, @id, date) |
||||
end |
||||
|
||||
def add(account_id, at_time = Time.now.utc) |
||||
Day.new(@prefix, @id, at_time).add(account_id) |
||||
end |
||||
|
||||
def aggregate(date_range) |
||||
Aggregate.new(@prefix, @id, date_range) |
||||
end |
||||
|
||||
def each(&block) |
||||
if block_given? |
||||
(0...7).map { |i| block.call(get(i.days.ago)) } |
||||
else |
||||
to_enum(:each) |
||||
end |
||||
end |
||||
|
||||
def as_json(*) |
||||
map(&:as_json) |
||||
end |
||||
end |
@ -0,0 +1,117 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Trends::Links < Trends::Base |
||||
PREFIX = 'trending_links' |
||||
|
||||
self.default_options = { |
||||
threshold: 15, |
||||
review_threshold: 10, |
||||
max_score_cooldown: 2.days.freeze, |
||||
max_score_halflife: 8.hours.freeze, |
||||
} |
||||
|
||||
def register(status, at_time = Time.now.utc) |
||||
original_status = status.reblog? ? status.reblog : status |
||||
|
||||
return unless original_status.public_visibility? && status.public_visibility? && |
||||
!original_status.account.silenced? && !status.account.silenced? && |
||||
!original_status.spoiler_text? |
||||
|
||||
original_status.preview_cards.each do |preview_card| |
||||
add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends? |
||||
end |
||||
end |
||||
|
||||
def add(preview_card, account_id, at_time = Time.now.utc) |
||||
preview_card.history.add(account_id, at_time) |
||||
record_used_id(preview_card.id, at_time) |
||||
end |
||||
|
||||
def get(allowed, limit) |
||||
preview_card_ids = currently_trending_ids(allowed, limit) |
||||
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) |
||||
preview_card_ids.map { |id| preview_cards[id] }.compact |
||||
end |
||||
|
||||
def refresh(at_time = Time.now.utc) |
||||
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) |
||||
calculate_scores(preview_cards, at_time) |
||||
trim_older_items |
||||
end |
||||
|
||||
def request_review |
||||
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) |
||||
|
||||
preview_cards_requiring_review = preview_cards.filter_map do |preview_card| |
||||
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? |
||||
|
||||
if preview_card.provider.nil? |
||||
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) |
||||
else |
||||
preview_card.provider.touch(:requested_review_at) |
||||
end |
||||
|
||||
preview_card |
||||
end |
||||
|
||||
return if preview_cards_requiring_review.empty? |
||||
|
||||
User.staff.includes(:account).find_each do |user| |
||||
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? |
||||
end |
||||
end |
||||
|
||||
protected |
||||
|
||||
def key_prefix |
||||
PREFIX |
||||
end |
||||
|
||||
private |
||||
|
||||
def calculate_scores(preview_cards, at_time) |
||||
preview_cards.each do |preview_card| |
||||
expected = preview_card.history.get(at_time - 1.day).accounts.to_f |
||||
expected = 1.0 if expected.zero? |
||||
observed = preview_card.history.get(at_time).accounts.to_f |
||||
max_time = preview_card.max_score_at |
||||
max_score = preview_card.max_score |
||||
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) |
||||
|
||||
score = begin |
||||
if expected > observed || observed < options[:threshold] |
||||
0 |
||||
else |
||||
((observed - expected)**2) / expected |
||||
end |
||||
end |
||||
|
||||
if score > max_score |
||||
max_score = score |
||||
max_time = at_time |
||||
|
||||
# Not interested in triggering any callbacks for this |
||||
preview_card.update_columns(max_score: max_score, max_score_at: max_time) |
||||
end |
||||
|
||||
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) |
||||
|
||||
if decaying_score.zero? |
||||
redis.zrem("#{PREFIX}:all", preview_card.id) |
||||
redis.zrem("#{PREFIX}:allowed", preview_card.id) |
||||
else |
||||
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) |
||||
|
||||
if preview_card.trendable? |
||||
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) |
||||
else |
||||
redis.zrem("#{PREFIX}:allowed", preview_card.id) |
||||
end |
||||
end |
||||
end |
||||
end |
||||
|
||||
def would_be_trending?(id) |
||||
score(id) > score_at_rank(options[:review_threshold] - 1) |
||||
end |
||||
end |
@ -0,0 +1,111 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Trends::Tags < Trends::Base |
||||
PREFIX = 'trending_tags' |
||||
|
||||
self.default_options = { |
||||
threshold: 15, |
||||
review_threshold: 10, |
||||
max_score_cooldown: 2.days.freeze, |
||||
max_score_halflife: 4.hours.freeze, |
||||
} |
||||
|
||||
def register(status, at_time = Time.now.utc) |
||||
original_status = status.reblog? ? status.reblog : status |
||||
|
||||
return unless original_status.public_visibility? && status.public_visibility? && |
||||
!original_status.account.silenced? && !status.account.silenced? |
||||
|
||||
original_status.tags.each do |tag| |
||||
add(tag, status.account_id, at_time) if tag.usable? |
||||
end |
||||
end |
||||
|
||||
def add(tag, account_id, at_time = Time.now.utc) |
||||
tag.history.add(account_id, at_time) |
||||
record_used_id(tag.id, at_time) |
||||
end |
||||
|
||||
def refresh(at_time = Time.now.utc) |
||||
tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) |
||||
calculate_scores(tags, at_time) |
||||
trim_older_items |
||||
end |
||||
|
||||
def get(allowed, limit) |
||||
tag_ids = currently_trending_ids(allowed, limit) |
||||
tags = Tag.where(id: tag_ids).index_by(&:id) |
||||
tag_ids.map { |id| tags[id] }.compact |
||||
end |
||||
|
||||
def request_review |
||||
tags = Tag.where(id: currently_trending_ids(false, -1)) |
||||
|
||||
tags_requiring_review = tags.filter_map do |tag| |
||||
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? |
||||
|
||||
tag.touch(:requested_review_at) |
||||
tag |
||||
end |
||||
|
||||
return if tags_requiring_review.empty? |
||||
|
||||
User.staff.includes(:account).find_each do |user| |
||||
AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? |
||||
end |
||||
end |
||||
|
||||
protected |
||||
|
||||
def key_prefix |
||||
PREFIX |
||||
end |
||||
|
||||
private |
||||
|
||||
def calculate_scores(tags, at_time) |
||||
tags.each do |tag| |
||||
expected = tag.history.get(at_time - 1.day).accounts.to_f |
||||
expected = 1.0 if expected.zero? |
||||
observed = tag.history.get(at_time).accounts.to_f |
||||
max_time = tag.max_score_at |
||||
max_score = tag.max_score |
||||
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) |
||||
|
||||
score = begin |
||||
if expected > observed || observed < options[:threshold] |
||||
0 |
||||
else |
||||
((observed - expected)**2) / expected |
||||
end |
||||
end |
||||
|
||||
if score > max_score |
||||
max_score = score |
||||
max_time = at_time |
||||
|
||||
# Not interested in triggering any callbacks for this |
||||
tag.update_columns(max_score: max_score, max_score_at: max_time) |
||||
end |
||||
|
||||
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) |
||||
|
||||
if decaying_score.zero? |
||||
redis.zrem("#{PREFIX}:all", tag.id) |
||||
redis.zrem("#{PREFIX}:allowed", tag.id) |
||||
else |
||||
redis.zadd("#{PREFIX}:all", decaying_score, tag.id) |
||||
|
||||
if tag.trendable? |
||||
redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) |
||||
else |
||||
redis.zrem("#{PREFIX}:allowed", tag.id) |
||||
end |
||||
end |
||||
end |
||||
end |
||||
|
||||
def would_be_trending?(id) |
||||
score(id) > score_at_rank(options[:review_threshold] - 1) |
||||
end |
||||
end |
@ -0,0 +1,11 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class PreviewCardPolicy < ApplicationPolicy |
||||
def index? |
||||
staff? |
||||
end |
||||
|
||||
def update? |
||||
staff? |
||||
end |
||||
end |
@ -0,0 +1,11 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class PreviewCardProviderPolicy < ApplicationPolicy |
||||
def index? |
||||
staff? |
||||
end |
||||
|
||||
def update? |
||||
staff? |
||||
end |
||||
end |
@ -0,0 +1,5 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class REST::Trends::LinkSerializer < REST::PreviewCardSerializer |
||||
attributes :history |
||||
end |
@ -1,19 +0,0 @@ |
||||
.batch-table__row |
||||
- if batch_available |
||||
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox |
||||
= f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id |
||||
|
||||
.directory__tag |
||||
= link_to admin_tag_path(tag.id) do |
||||
%h4 |
||||
= fa_icon 'hashtag' |
||||
= tag.name |
||||
|
||||
%small |
||||
= t('admin.tags.unique_uses_today', count: tag.history.first[:accounts]) |
||||
|
||||
- if tag.trending? |
||||
= fa_icon 'fire fw' |
||||
= t('admin.tags.trending_right_now') |
||||
|
||||
.trends__item__current= friendly_number_to_human tag.history.first[:uses] |
@ -1,74 +0,0 @@ |
||||
- content_for :page_title do |
||||
= t('admin.tags.title') |
||||
|
||||
- content_for :header_tags do |
||||
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' |
||||
|
||||
.filters |
||||
.filter-subset |
||||
%strong= t('admin.tags.review') |
||||
%ul |
||||
%li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil |
||||
%li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil |
||||
%li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil |
||||
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil |
||||
|
||||
.filter-subset |
||||
%strong= t('generic.order_by') |
||||
%ul |
||||
%li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil |
||||
%li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil |
||||
%li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil |
||||
|
||||
|
||||
= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do |
||||
.fields-group |
||||
- TagFilter::KEYS.each do |key| |
||||
= hidden_field_tag key, params[key] if params[key].present? |
||||
|
||||
- %i(name).each do |key| |
||||
.input.string.optional |
||||
= text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}") |
||||
|
||||
.actions |
||||
%button.button= t('admin.accounts.search') |
||||
= link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative' |
||||
|
||||
%hr.spacer/ |
||||
|
||||
= form_for(@form, url: batch_admin_tags_path) do |f| |
||||
= hidden_field_tag :page, params[:page] || 1 |
||||
|
||||
- TagFilter::KEYS.each do |key| |
||||
= hidden_field_tag key, params[key] if params[key].present? |
||||
|
||||
.batch-table.optional |
||||
.batch-table__toolbar |
||||
- if params[:pending_review] == '1' || params[:unreviewed] == '1' |
||||
%label.batch-table__toolbar__select.batch-checkbox-all |
||||
= check_box_tag :batch_checkbox_all, nil, false |
||||
.batch-table__toolbar__actions |
||||
= f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
|
||||
= f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
- else |
||||
.batch-table__toolbar__actions |
||||
%span.neutral-hint= t('generic.no_batch_actions_available') |
||||
|
||||
.batch-table__body |
||||
- if @tags.empty? |
||||
= nothing_here 'nothing-here--under-tabs' |
||||
- else |
||||
= render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' } |
||||
|
||||
= paginate @tags |
||||
|
||||
- if params[:pending_review] == '1' || params[:unreviewed] == '1' |
||||
%hr.spacer/ |
||||
|
||||
%div.action-buttons |
||||
%div |
||||
= link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button' |
||||
|
||||
%div |
||||
= link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive' |
@ -0,0 +1,30 @@ |
||||
.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] } |
||||
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox |
||||
= f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id |
||||
|
||||
.batch-table__row__content.pending-account |
||||
.pending-account__header |
||||
= link_to preview_card.title, preview_card.url |
||||
|
||||
%br/ |
||||
|
||||
- if preview_card.provider_name.present? |
||||
= preview_card.provider_name |
||||
• |
||||
|
||||
- if preview_card.language.present? |
||||
= human_locale(preview_card.language) |
||||
• |
||||
|
||||
= t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts }) |
||||
|
||||
- if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id)) |
||||
• |
||||
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) |
||||
|
||||
- if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago |
||||
• |
||||
= t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short)) |
||||
- elsif preview_card.provider&.requires_review? |
||||
• |
||||
= t('admin.trends.pending_review') |
@ -0,0 +1,41 @@ |
||||
- content_for :page_title do |
||||
= t('admin.trends.links.title') |
||||
|
||||
- content_for :header_tags do |
||||
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' |
||||
|
||||
.filters |
||||
.filter-subset |
||||
%strong= t('admin.trends.trending') |
||||
%ul |
||||
%li= filter_link_to t('generic.all'), trending: nil |
||||
%li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' |
||||
.back-link |
||||
= link_to admin_trends_links_preview_card_providers_path do |
||||
= t('admin.trends.preview_card_providers.title') |
||||
= fa_icon 'chevron-right fw' |
||||
|
||||
%hr.spacer/ |
||||
|
||||
= form_for(@form, url: batch_admin_trends_links_path) do |f| |
||||
= hidden_field_tag :page, params[:page] || 1 |
||||
|
||||
- PreviewCardFilter::KEYS.each do |key| |
||||
= hidden_field_tag key, params[key] if params[key].present? |
||||
|
||||
.batch-table |
||||
.batch-table__toolbar |
||||
%label.batch-table__toolbar__select.batch-checkbox-all |
||||
= check_box_tag :batch_checkbox_all, nil, false |
||||
.batch-table__toolbar__actions |
||||
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
= f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
= f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
.batch-table__body |
||||
- if @preview_cards.empty? |
||||
= nothing_here 'nothing-here--under-tabs' |
||||
- else |
||||
= render partial: 'preview_card', collection: @preview_cards, locals: { f: f } |
||||
|
||||
= paginate @preview_cards |
@ -0,0 +1,16 @@ |
||||
.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] } |
||||
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox |
||||
= f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id |
||||
|
||||
.batch-table__row__content.pending-account |
||||
.pending-account__header |
||||
%strong= preview_card_provider.domain |
||||
|
||||
%br/ |
||||
|
||||
- if preview_card_provider.requires_review? |
||||
= t('admin.trends.pending_review') |
||||
- elsif preview_card_provider.trendable? |
||||
= t('admin.trends.preview_card_providers.allowed') |
||||
- else |
||||
= t('admin.trends.preview_card_providers.rejected') |
@ -0,0 +1,43 @@ |
||||
- content_for :page_title do |
||||
= t('admin.trends.preview_card_providers.title') |
||||
|
||||
- content_for :header_tags do |
||||
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' |
||||
|
||||
.filters |
||||
.filter-subset |
||||
%strong= t('admin.tags.review') |
||||
%ul |
||||
%li= filter_link_to t('generic.all'), status: nil |
||||
%li= filter_link_to t('admin.trends.approved'), status: 'approved' |
||||
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected' |
||||
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review' |
||||
.back-link |
||||
= link_to admin_trends_links_path do |
||||
= fa_icon 'chevron-left fw' |
||||
= t('admin.trends.links.title') |
||||
|
||||
|
||||
%hr.spacer/ |
||||
|
||||
= form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f| |
||||
= hidden_field_tag :page, params[:page] || 1 |
||||
|
||||
- PreviewCardProviderFilter::KEYS.each do |key| |
||||
= hidden_field_tag key, params[key] if params[key].present? |
||||
|
||||
.batch-table.optional |
||||
.batch-table__toolbar |
||||
%label.batch-table__toolbar__select.batch-checkbox-all |
||||
= check_box_tag :batch_checkbox_all, nil, false |
||||
.batch-table__toolbar__actions |
||||
= f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
= f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
|
||||
.batch-table__body |
||||
- if @preview_card_providers.empty? |
||||
= nothing_here 'nothing-here--under-tabs' |
||||
- else |
||||
= render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f } |
||||
|
||||
= paginate @preview_card_providers |
@ -0,0 +1,24 @@ |
||||
.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] } |
||||
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox |
||||
= f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id |
||||
|
||||
.batch-table__row__content.pending-account |
||||
.pending-account__header |
||||
= link_to admin_tag_path(tag.id) do |
||||
= fa_icon 'hashtag' |
||||
= tag.name |
||||
|
||||
%br/ |
||||
|
||||
= t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts }) |
||||
|
||||
- if tag.trendable? && (rank = Trends.tags.rank(tag.id)) |
||||
• |
||||
%abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) |
||||
|
||||
- if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago |
||||
• |
||||
= t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short)) |
||||
- elsif tag.requires_review? |
||||
• |
||||
= t('admin.trends.pending_review') |
@ -0,0 +1,38 @@ |
||||
- content_for :page_title do |
||||
= t('admin.trends.tags.title') |
||||
|
||||
- content_for :header_tags do |
||||
= javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' |
||||
|
||||
.filters |
||||
.filter-subset |
||||
%strong= t('admin.tags.review') |
||||
%ul |
||||
%li= filter_link_to t('generic.all'), status: nil |
||||
%li= filter_link_to t('admin.trends.approved'), status: 'approved' |
||||
%li= filter_link_to t('admin.trends.rejected'), status: 'rejected' |
||||
%li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review' |
||||
|
||||
%hr.spacer/ |
||||
|
||||
= form_for(@form, url: batch_admin_trends_tags_path) do |f| |
||||
= hidden_field_tag :page, params[:page] || 1 |
||||
|
||||
- TagFilter::KEYS.each do |key| |
||||
= hidden_field_tag key, params[key] if params[key].present? |
||||
|
||||
.batch-table.optional |
||||
.batch-table__toolbar |
||||
%label.batch-table__toolbar__select.batch-checkbox-all |
||||
= check_box_tag :batch_checkbox_all, nil, false |
||||
.batch-table__toolbar__actions |
||||
= f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
= f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } |
||||
|
||||
.batch-table__body |
||||
- if @tags.empty? |
||||
= nothing_here 'nothing-here--under-tabs' |
||||
- else |
||||
= render partial: 'tag', collection: @tags, locals: { f: f } |
||||
|
||||
= paginate @tags |
@ -0,0 +1,16 @@ |
||||
<%= raw t('application_mailer.salutation', name: display_name(@me)) %> |
||||
|
||||
<%= raw t('admin_mailer.new_trending_links.body') %> |
||||
|
||||
<% @links.each do |link| %> |
||||
- <%= link.title %> • <%= link.url %> |
||||
<%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> |
||||
<% end %> |
||||
|
||||
<% if @lowest_trending_link %> |
||||
<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %> |
||||
<% else %> |
||||
<%= t('admin_mailer.new_trending_links.no_approved_links') %> |
||||
<% end %> |
||||
|
||||
<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> |
@ -1,5 +0,0 @@ |
||||
<%= raw t('application_mailer.salutation', name: display_name(@me)) %> |
||||
|
||||
<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %> |
||||
|
||||
<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %> |
@ -0,0 +1,16 @@ |
||||
<%= raw t('application_mailer.salutation', name: display_name(@me)) %> |
||||
|
||||
<%= raw t('admin_mailer.new_trending_tags.body') %> |
||||
|
||||
<% @tags.each do |tag| %> |
||||
- #<%= tag.name %> |
||||
<%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> |
||||
<% end %> |
||||
|
||||
<% if @lowest_trending_tag %> |
||||
<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %> |
||||
<% else %> |
||||
<%= t('admin_mailer.new_trending_tags.no_approved_tags') %> |
||||
<% end %> |
||||
|
||||
<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %> |
@ -1,11 +1,11 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Scheduler::TrendingTagsScheduler |
||||
class Scheduler::Trends::RefreshScheduler |
||||
include Sidekiq::Worker |
||||
|
||||
sidekiq_options retry: 0 |
||||
|
||||
def perform |
||||
TrendingTags.update! if Setting.trends |
||||
Trends.refresh! |
||||
end |
||||
end |
@ -0,0 +1,11 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class Scheduler::Trends::ReviewNotificationsScheduler |
||||
include Sidekiq::Worker |
||||
|
||||
sidekiq_options retry: 0 |
||||
|
||||
def perform |
||||
Trends.request_review! |
||||
end |
||||
end |
@ -0,0 +1,12 @@ |
||||
class CreatePreviewCardProviders < ActiveRecord::Migration[6.1] |
||||
def change |
||||
create_table :preview_card_providers do |t| |
||||
t.string :domain, null: false, default: '', index: { unique: true } |
||||
t.attachment :icon |
||||
t.boolean :trendable |
||||
t.datetime :reviewed_at |
||||
t.datetime :requested_review_at |
||||
t.timestamps |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,7 @@ |
||||
class AddLanguageToPreviewCards < ActiveRecord::Migration[6.1] |
||||
def change |
||||
add_column :preview_cards, :language, :string |
||||
add_column :preview_cards, :max_score, :float |
||||
add_column :preview_cards, :max_score_at, :datetime |
||||
end |
||||
end |
@ -0,0 +1,5 @@ |
||||
class AddTrendableToPreviewCards < ActiveRecord::Migration[6.1] |
||||
def change |
||||
add_column :preview_cards, :trendable, :boolean |
||||
end |
||||
end |
@ -0,0 +1,5 @@ |
||||
class AddLinkTypeToPreviewCards < ActiveRecord::Migration[6.1] |
||||
def change |
||||
add_column :preview_cards, :link_type, :int |
||||
end |
||||
end |
@ -0,0 +1,22 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe Api::V1::Trends::TagsController, type: :controller do |
||||
render_views |
||||
|
||||
describe 'GET #index' do |
||||
before do |
||||
trending_tags = double() |
||||
|
||||
allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag)) |
||||
allow(Trends).to receive(:tags).and_return(trending_tags) |
||||
|
||||
get :index |
||||
end |
||||
|
||||
it 'returns http success' do |
||||
expect(response).to have_http_status(200) |
||||
end |
||||
end |
||||
end |
@ -1,18 +0,0 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe Api::V1::TrendsController, type: :controller do |
||||
render_views |
||||
|
||||
describe 'GET #index' do |
||||
before do |
||||
allow(TrendingTags).to receive(:get).and_return(Fabricate.times(10, :tag)) |
||||
get :index |
||||
end |
||||
|
||||
it 'returns http success' do |
||||
expect(response).to have_http_status(200) |
||||
end |
||||
end |
||||
end |
@ -1,68 +0,0 @@ |
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe TrendingTags do |
||||
describe '.record_use!' do |
||||
pending |
||||
end |
||||
|
||||
describe '.update!' do |
||||
let!(:at_time) { Time.now.utc } |
||||
let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) } |
||||
let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) } |
||||
let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) } |
||||
|
||||
before do |
||||
allow(Redis.current).to receive(:pfcount) do |key| |
||||
case key |
||||
when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" |
||||
2 |
||||
when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts" |
||||
16 |
||||
when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" |
||||
0 |
||||
when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts" |
||||
4 |
||||
when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" |
||||
13 |
||||
end |
||||
end |
||||
|
||||
Redis.current.zadd('trending_tags', 0.9, tag3.id) |
||||
Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id]) |
||||
|
||||
tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours) |
||||
|
||||
described_class.update!(at_time) |
||||
end |
||||
|
||||
it 'calculates and re-calculates scores' do |
||||
expect(described_class.get(10, filtered: false)).to eq [tag1, tag3] |
||||
end |
||||
|
||||
it 'omits hashtags below threshold' do |
||||
expect(described_class.get(10, filtered: false)).to_not include(tag2) |
||||
end |
||||
|
||||
it 'decays scores' do |
||||
expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9 |
||||
end |
||||
end |
||||
|
||||
describe '.trending?' do |
||||
let(:tag) { Fabricate(:tag) } |
||||
|
||||
before do |
||||
10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) } |
||||
end |
||||
|
||||
it 'returns true if the hashtag is within limit' do |
||||
Redis.current.zadd('trending_tags', 11, tag.id) |
||||
expect(described_class.trending?(tag)).to be true |
||||
end |
||||
|
||||
it 'returns false if the hashtag is outside the limit' do |
||||
Redis.current.zadd('trending_tags', 0, tag.id) |
||||
expect(described_class.trending?(tag)).to be false |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,67 @@ |
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe Trends::Tags do |
||||
subject { described_class.new(threshold: 5, review_threshold: 10) } |
||||
|
||||
let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) } |
||||
|
||||
describe '#add' do |
||||
let(:tag) { Fabricate(:tag) } |
||||
|
||||
before do |
||||
subject.add(tag, 1, at_time) |
||||
end |
||||
|
||||
it 'records history' do |
||||
expect(tag.history.get(at_time).accounts).to eq 1 |
||||
end |
||||
|
||||
it 'records use' do |
||||
expect(subject.send(:recently_used_ids, at_time)).to eq [tag.id] |
||||
end |
||||
end |
||||
|
||||
describe '#get' do |
||||
pending |
||||
end |
||||
|
||||
describe '#refresh' do |
||||
let!(:today) { at_time } |
||||
let!(:yesterday) { today - 1.day } |
||||
|
||||
let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) } |
||||
let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) } |
||||
let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) } |
||||
|
||||
before do |
||||
2.times { |i| subject.add(tag1, i, yesterday) } |
||||
13.times { |i| subject.add(tag3, i, yesterday) } |
||||
16.times { |i| subject.add(tag1, i, today) } |
||||
4.times { |i| subject.add(tag2, i, today) } |
||||
end |
||||
|
||||
context do |
||||
before do |
||||
subject.refresh(yesterday + 12.hours) |
||||
subject.refresh(at_time) |
||||
end |
||||
|
||||
it 'calculates and re-calculates scores' do |
||||
expect(subject.get(false, 10)).to eq [tag1, tag3] |
||||
end |
||||
|
||||
it 'omits hashtags below threshold' do |
||||
expect(subject.get(false, 10)).to_not include(tag2) |
||||
end |
||||
end |
||||
|
||||
it 'decays scores' do |
||||
subject.refresh(yesterday + 12.hours) |
||||
original_score = subject.score(tag3.id) |
||||
expect(original_score).to eq 144.0 |
||||
subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife]) |
||||
decayed_score = subject.score(tag3.id) |
||||
expect(decayed_score).to be <= original_score / 2 |
||||
end |
||||
end |
||||
end |
Loading…
Reference in new issue