Allow import/export of instance-level domain blocks/allows (#1754)
* Allow import/export of instance-level domain blocks/allows. Fixes #15095 * Pacify circleci * Address simple code review feedback * Add headers to exported CSV * Extract common import/export functionality to AdminExportControllerConcern * Add additional fields to instance-blocked domain export * Address review feedback * Split instance domain block/allow import/export into separate pages/controllers * Address code review feedback * Pacify DeepSource * Work around Paperclip::HasAttachmentFile for Rails 6 * Fix deprecated API warning in export tests * Remove after_commit workaroundlocal
parent
dc350be6f5
commit
94e98864e3
16 changed files with 414 additions and 1 deletions
@ -0,0 +1,60 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'csv' |
||||
|
||||
module Admin |
||||
class ExportDomainAllowsController < BaseController |
||||
include AdminExportControllerConcern |
||||
|
||||
before_action :set_dummy_import!, only: [:new] |
||||
|
||||
ROWS_PROCESSING_LIMIT = 20_000 |
||||
|
||||
def new |
||||
authorize :domain_allow, :create? |
||||
end |
||||
|
||||
def export |
||||
authorize :instance, :index? |
||||
send_export_file |
||||
end |
||||
|
||||
def import |
||||
authorize :domain_allow, :create? |
||||
begin |
||||
@import = Admin::Import.new(import_params) |
||||
parse_import_data!(export_headers) |
||||
|
||||
@data.take(ROWS_PROCESSING_LIMIT).each do |row| |
||||
domain = row['#domain'].strip |
||||
next if DomainAllow.allowed?(domain) |
||||
|
||||
domain_allow = DomainAllow.new(domain: domain) |
||||
log_action :create, domain_allow if domain_allow.save |
||||
end |
||||
flash[:notice] = I18n.t('admin.domain_allows.created_msg') |
||||
rescue ActionController::ParameterMissing |
||||
flash[:error] = I18n.t('admin.export_domain_allows.no_file') |
||||
end |
||||
redirect_to admin_instances_path |
||||
end |
||||
|
||||
private |
||||
|
||||
def export_filename |
||||
'domain_allows.csv' |
||||
end |
||||
|
||||
def export_headers |
||||
%w(#domain) |
||||
end |
||||
|
||||
def export_data |
||||
CSV.generate(headers: export_headers, write_headers: true) do |content| |
||||
DomainAllow.allowed_domains.each do |instance| |
||||
content << [instance.domain] |
||||
end |
||||
end |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,68 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
require 'csv' |
||||
|
||||
module Admin |
||||
class ExportDomainBlocksController < BaseController |
||||
include AdminExportControllerConcern |
||||
|
||||
before_action :set_dummy_import!, only: [:new] |
||||
|
||||
ROWS_PROCESSING_LIMIT = 20_000 |
||||
|
||||
def new |
||||
authorize :domain_block, :create? |
||||
end |
||||
|
||||
def export |
||||
authorize :instance, :index? |
||||
send_export_file |
||||
end |
||||
|
||||
def import |
||||
authorize :domain_block, :create? |
||||
begin |
||||
@import = Admin::Import.new(import_params) |
||||
parse_import_data!(export_headers) |
||||
|
||||
@data.take(ROWS_PROCESSING_LIMIT).each do |row| |
||||
domain = row['#domain'].strip |
||||
next if DomainBlock.rule_for(domain).present? |
||||
|
||||
domain_block = DomainBlock.new(domain: domain, |
||||
severity: row['#severity'].strip, |
||||
reject_media: row['#reject_media'].strip, |
||||
reject_reports: row['#reject_reports'].strip, |
||||
public_comment: row['#public_comment'].strip, |
||||
obfuscate: row['#obfuscate'].strip) |
||||
if domain_block.save |
||||
DomainBlockWorker.perform_async(domain_block.id) |
||||
log_action :create, domain_block |
||||
end |
||||
end |
||||
flash[:notice] = I18n.t('admin.domain_blocks.created_msg') |
||||
rescue ActionController::ParameterMissing |
||||
flash[:error] = I18n.t('admin.export_domain_blocks.no_file') |
||||
end |
||||
redirect_to admin_instances_path(limited: '1') |
||||
end |
||||
|
||||
private |
||||
|
||||
def export_filename |
||||
'domain_blocks.csv' |
||||
end |
||||
|
||||
def export_headers |
||||
%w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate) |
||||
end |
||||
|
||||
def export_data |
||||
CSV.generate(headers: export_headers, write_headers: true) do |content| |
||||
DomainBlock.with_user_facing_limitations.each do |instance| |
||||
content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate] |
||||
end |
||||
end |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,39 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
module AdminExportControllerConcern |
||||
extend ActiveSupport::Concern |
||||
|
||||
private |
||||
|
||||
def send_export_file |
||||
respond_to do |format| |
||||
format.csv { send_data export_data, filename: export_filename } |
||||
end |
||||
end |
||||
|
||||
def export_data |
||||
raise 'Override in controller' |
||||
end |
||||
|
||||
def export_filename |
||||
raise 'Override in controller' |
||||
end |
||||
|
||||
def set_dummy_import! |
||||
@import = Admin::Import.new |
||||
end |
||||
|
||||
def import_params |
||||
params.require(:admin_import).permit(:data) |
||||
end |
||||
|
||||
def import_data |
||||
Paperclip.io_adapters.for(@import.data).read |
||||
end |
||||
|
||||
def parse_import_data!(default_headers) |
||||
data = CSV.parse(import_data, headers: true) |
||||
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(default_headers[0]) |
||||
@data = data.reject(&:blank?) |
||||
end |
||||
end |
@ -0,0 +1,29 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
# A non-activerecord helper class for csv upload |
||||
class Admin::Import |
||||
extend ActiveModel::Callbacks |
||||
include ActiveModel::Model |
||||
include Paperclip::Glue |
||||
|
||||
FILE_TYPES = %w(text/plain text/csv application/csv).freeze |
||||
|
||||
# Paperclip required callbacks |
||||
define_model_callbacks :save, only: [:after] |
||||
define_model_callbacks :destroy, only: [:before, :after] |
||||
|
||||
attr_accessor :data_file_name, :data_content_type |
||||
|
||||
has_attached_file :data |
||||
validates_attachment_content_type :data, content_type: FILE_TYPES |
||||
validates_attachment_presence :data |
||||
validates_with AdminImportValidator, on: :create |
||||
|
||||
def save |
||||
run_callbacks :save |
||||
end |
||||
|
||||
def destroy |
||||
run_callbacks :destroy |
||||
end |
||||
end |
@ -0,0 +1,19 @@ |
||||
# frozen_string_literal: true |
||||
|
||||
class AdminImportValidator < ActiveModel::Validator |
||||
FIRST_HEADER = '#domain' |
||||
|
||||
def validate(import) |
||||
return if import.type.blank? || import.data.blank? |
||||
|
||||
# We parse because newlines could be part of individual rows. This |
||||
# runs on create so we should be reading the local file here before |
||||
# it is uploaded to object storage or moved anywhere... |
||||
csv_data = CSV.parse(import.data.queued_for_write[:original].read) |
||||
|
||||
row_count = csv_data.size |
||||
row_count -= 1 if csv_data.first&.first == FIRST_HEADER |
||||
|
||||
import.errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT)) if row_count > Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT |
||||
end |
||||
end |
@ -0,0 +1,10 @@ |
||||
- content_for :page_title do |
||||
= t('.title') |
||||
|
||||
= simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f| |
||||
.fields-row |
||||
.fields-group.fields-row__column.fields-row__column-6 |
||||
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file |
||||
|
||||
.actions |
||||
= f.button :button, t('imports.upload'), type: :submit |
@ -0,0 +1,10 @@ |
||||
- content_for :page_title do |
||||
= t('.title') |
||||
|
||||
= simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f| |
||||
.fields-row |
||||
.fields-group.fields-row__column.fields-row__column-6 |
||||
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file |
||||
|
||||
.actions |
||||
= f.button :button, t('imports.upload'), type: :submit |
@ -0,0 +1,48 @@ |
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe Admin::DomainAllowsController, type: :controller do |
||||
render_views |
||||
|
||||
before do |
||||
sign_in Fabricate(:user, admin: true), scope: :user |
||||
end |
||||
|
||||
describe 'GET #new' do |
||||
it 'assigns a new domain allow' do |
||||
get :new |
||||
|
||||
expect(assigns(:domain_allow)).to be_instance_of(DomainAllow) |
||||
expect(response).to have_http_status(200) |
||||
end |
||||
end |
||||
|
||||
describe 'POST #create' do |
||||
it 'blocks the domain when succeeded to save' do |
||||
post :create, params: { domain_allow: { domain: 'example.com' } } |
||||
|
||||
expect(flash[:notice]).to eq I18n.t('admin.domain_allows.created_msg') |
||||
expect(response).to redirect_to(admin_instances_path) |
||||
end |
||||
|
||||
it 'renders new when failed to save' do |
||||
Fabricate(:domain_allow, domain: 'example.com') |
||||
|
||||
post :create, params: { domain_allow: { domain: 'example.com' } } |
||||
|
||||
expect(response).to render_template :new |
||||
end |
||||
end |
||||
|
||||
describe 'DELETE #destroy' do |
||||
it 'disallows the domain' do |
||||
service = double(call: true) |
||||
allow(UnallowDomainService).to receive(:new).and_return(service) |
||||
domain_allow = Fabricate(:domain_allow) |
||||
delete :destroy, params: { id: domain_allow.id } |
||||
|
||||
expect(service).to have_received(:call).with(domain_allow) |
||||
expect(flash[:notice]).to eq I18n.t('admin.domain_allows.destroyed_msg') |
||||
expect(response).to redirect_to(admin_instances_path) |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,42 @@ |
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe Admin::ExportDomainAllowsController, type: :controller do |
||||
render_views |
||||
|
||||
before do |
||||
sign_in Fabricate(:user, admin: true), scope: :user |
||||
end |
||||
|
||||
describe 'GET #export' do |
||||
it 'renders instances' do |
||||
Fabricate(:domain_allow, domain: 'good.domain') |
||||
Fabricate(:domain_allow, domain: 'better.domain') |
||||
|
||||
get :export, params: { format: :csv } |
||||
expect(response).to have_http_status(200) |
||||
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv'))) |
||||
end |
||||
end |
||||
|
||||
describe 'POST #import' do |
||||
it 'allows imported domains' do |
||||
post :import, params: { admin_import: { data: fixture_file_upload('domain_allows.csv') } } |
||||
|
||||
expect(response).to redirect_to(admin_instances_path) |
||||
|
||||
# Header should not be imported |
||||
expect(DomainAllow.where(domain: '#domain').present?).to eq(false) |
||||
|
||||
# Domains should now be added |
||||
get :export, params: { format: :csv } |
||||
expect(response).to have_http_status(200) |
||||
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv'))) |
||||
end |
||||
|
||||
it 'displays error on no file selected' do |
||||
post :import, params: { admin_import: {} } |
||||
expect(response).to redirect_to(admin_instances_path) |
||||
expect(flash[:error]).to eq(I18n.t('admin.export_domain_allows.no_file')) |
||||
end |
||||
end |
||||
end |
@ -0,0 +1,47 @@ |
||||
require 'rails_helper' |
||||
|
||||
RSpec.describe Admin::ExportDomainBlocksController, type: :controller do |
||||
render_views |
||||
|
||||
before do |
||||
sign_in Fabricate(:user, admin: true), scope: :user |
||||
end |
||||
|
||||
describe 'GET #export' do |
||||
it 'renders instances' do |
||||
Fabricate(:domain_block, domain: 'bad.domain', severity: 'silence', public_comment: 'bad') |
||||
Fabricate(:domain_block, domain: 'worse.domain', severity: 'suspend', reject_media: true, reject_reports: true, public_comment: 'worse', obfuscate: true) |
||||
Fabricate(:domain_block, domain: 'reject.media', severity: 'noop', reject_media: true, public_comment: 'reject media') |
||||
Fabricate(:domain_block, domain: 'no.op', severity: 'noop', public_comment: 'noop') |
||||
|
||||
get :export, params: { format: :csv } |
||||
expect(response).to have_http_status(200) |
||||
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_blocks.csv'))) |
||||
end |
||||
end |
||||
|
||||
describe 'POST #import' do |
||||
it 'blocks imported domains' do |
||||
allow(DomainBlockWorker).to receive(:perform_async).and_return(true) |
||||
|
||||
post :import, params: { admin_import: { data: fixture_file_upload('domain_blocks.csv') } } |
||||
|
||||
expect(response).to redirect_to(admin_instances_path(limited: '1')) |
||||
expect(DomainBlockWorker).to have_received(:perform_async).exactly(3).times |
||||
|
||||
# Header should not be imported |
||||
expect(DomainBlock.where(domain: '#domain').present?).to eq(false) |
||||
|
||||
# Domains should now be added |
||||
get :export, params: { format: :csv } |
||||
expect(response).to have_http_status(200) |
||||
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_blocks.csv'))) |
||||
end |
||||
end |
||||
|
||||
it 'displays error on no file selected' do |
||||
post :import, params: { admin_import: {} } |
||||
expect(response).to redirect_to(admin_instances_path(limited: '1')) |
||||
expect(flash[:error]).to eq(I18n.t('admin.export_domain_blocks.no_file')) |
||||
end |
||||
end |
unable to load file from base commit
|
unable to load file from base commit
|
Loading…
Reference in new issue