From c3653bd1a4e3a67f95c9f1a726c7fe1ec7f10834 Mon Sep 17 00:00:00 2001 From: Mark Bussey Date: Mon, 29 Aug 2022 14:51:13 -0500 Subject: [PATCH] Add Background processing and UI Forms for CSV exports (#198) This change adds the scaffolding to let an administrator: 1. Select 1-3 objects for a CSV export 2. Submit an Export job 3. Run the job in the background 4. View the results 5. Download the exported file We expect to provide a different UX for selecting items for export, so the submission form is very bare-bones but functional. The status (show) page for exports need additional design but provides enough functionality to download the exported CSV for testing. --- app/controllers/exports_controller.rb | 20 +++++++++- app/jobs/batch_export_job.rb | 10 +++++ app/lib/tenejo/csv_exporter.rb | 20 ++++++++-- app/views/exports/new.html.erb | 32 ++++++++++++++++ spec/lib/tenejo/csv_exporter_spec.rb | 10 ++--- spec/requests/export_request_spec.rb | 53 +++++++++++++++++++++++++++ spec/routing/exports_routing_spec.rb | 41 +++++++++++++++++++++ 7 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 app/jobs/batch_export_job.rb create mode 100644 app/views/exports/new.html.erb create mode 100644 spec/requests/export_request_spec.rb create mode 100644 spec/routing/exports_routing_spec.rb diff --git a/app/controllers/exports_controller.rb b/app/controllers/exports_controller.rb index 3d86d6bf..31a0ef70 100644 --- a/app/controllers/exports_controller.rb +++ b/app/controllers/exports_controller.rb @@ -8,12 +8,28 @@ def index end def new - super - redirect_to new_export_path + @job = Export.new end def show super add_breadcrumb "##{@job.id} - #{I18n.t('tenejo.admin.sidebar.exports')}", @job end + + def create + @job = Export.new(job_params.merge({ user: current_user, status: :submitted })) + respond_to do |format| + if @job.save + BatchExportJob.perform_later(@job) + format.html { redirect_to @job } + format.json { render :show, status: :created, location: @job } + end + end + end + + private + + def job_params + params.require(:export).permit(identifiers: []) + end end diff --git a/app/jobs/batch_export_job.rb b/app/jobs/batch_export_job.rb new file mode 100644 index 00000000..e7b05c75 --- /dev/null +++ b/app/jobs/batch_export_job.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true +class BatchExportJob < ApplicationJob + queue_as :default + + def perform(export_job) + # probably not optimal, but enough until we get + # import hammered out + Tenejo::CsvExporter.new(export_job).run + end +end diff --git a/app/lib/tenejo/csv_exporter.rb b/app/lib/tenejo/csv_exporter.rb index 5f3ad493..1fb6b9e4 100644 --- a/app/lib/tenejo/csv_exporter.rb +++ b/app/lib/tenejo/csv_exporter.rb @@ -4,22 +4,34 @@ module Tenejo class CsvExporter - EXPORT_HEADERS = ([:primary_identifier, :error, :class, :title] + Tenejo::CsvImporter.collection_attributes_to_copy.keys + Tenejo::CsvImporter.work_attributes_to_copy.keys).uniq + EXCLUDE_FROM_EXPORT = [:date_modified, :identifier, :label, :arkivo_checksum, :state].freeze + HEADER_ROW = (([:primary_identifier, :error, :class, :title] \ + + Tenejo::CsvImporter.collection_attributes_to_copy.keys \ + + Tenejo::CsvImporter.work_attributes_to_copy.keys).uniq \ + - EXCLUDE_FROM_EXPORT).freeze def initialize(export_job) @export = export_job end def run + @export.status = :in_progress + @export.save output = CSV.generate(encoding: 'UTF-8', write_headers: true) do |csv| - csv << EXPORT_HEADERS # Header row + csv << HEADER_ROW csv << CSV::Row.new([:primary_identifier, :error], ["missing", "No identifiers provided"]) if @export.identifiers.empty? @export.identifiers.each do |id| csv << serialize(id) end end + # TODO: remove this after refactoring Tenejo metadata to rename primary_identifier and identifer + output.gsub!('primary_identifier,error,class,', 'identifier,error,object type,') + @export.manifest.attach(io: StringIO.new(output), filename: export_name, content_type: 'text/csv') + @export.status = :complete + @export.completed_at = Time.current + @export.save end private @@ -37,8 +49,8 @@ def export_name def serialize(id) obj = ActiveFedora::Base.where(primary_identifier_ssi: id).last return CSV::Row.new([:primary_identifier, :error], [id, "No match for identifier"]) unless obj - values = EXPORT_HEADERS.map { |attr| pack_field(obj.try(attr)) } - CSV::Row.new(EXPORT_HEADERS, values) + values = HEADER_ROW.map { |attr| pack_field(obj.try(attr)) } + CSV::Row.new(HEADER_ROW, values) end # Handle multi-value fields and normalize empty fields regardless of underlying class diff --git a/app/views/exports/new.html.erb b/app/views/exports/new.html.erb new file mode 100644 index 00000000..402c92e6 --- /dev/null +++ b/app/views/exports/new.html.erb @@ -0,0 +1,32 @@ +

New <%= @job.class %>

+ +<%= form_with(model: @job, local: true) do |form| %> + <% if @job.errors.any? %> +
+

<%= pluralize(@job.errors.count, "error") %> prohibited this job from being saved:

+ + +
+ <% end %> + +
+ <%= form.label :identifiers %> + <%= form.text_field :identifiers, id: 'export_identifiers_0', value: @job.identifiers[0], multiple: true %> +
+
+ <%= form.label :identifiers %> + <%= form.text_field :identifiers, id: 'export_identifiers_1', value: @job.identifiers[1], multiple: true %> +
+
+ <%= form.label :identifiers %> + <%= form.text_field :identifiers, id: 'export_identifiers_2', value: @job.identifiers[2], multiple: true %> +
+
+
+ <%= form.submit 'Submit', type: 'submit' %> +
+<% end %> diff --git a/spec/lib/tenejo/csv_exporter_spec.rb b/spec/lib/tenejo/csv_exporter_spec.rb index 617d6700..0214e9b1 100644 --- a/spec/lib/tenejo/csv_exporter_spec.rb +++ b/spec/lib/tenejo/csv_exporter_spec.rb @@ -35,7 +35,7 @@ export.save described_class.new(export).run rows = CSV.parse(export.manifest.download, headers: true) - expect(rows.first['primary_identifier']).to eq 'invalid_id' + expect(rows.first['identifier']).to eq 'invalid_id' expect(rows.first['error']).to eq 'No match for identifier' end @@ -48,16 +48,16 @@ rows = CSV.parse(export.manifest.download, headers: true) # Collection COL001 - expect(rows[0]['primary_identifier']).to eq 'COL001' + expect(rows[0]['identifier']).to eq 'COL001' expect(rows[0]['error']).to be_nil expect(rows[0]['title']).to include 'Test collection' - expect(rows[0]['class']).to eq "Collection" + expect(rows[0]['object type']).to eq "Collection" # Work WRK001 - expect(rows[1]['primary_identifier']).to eq 'WRK001' + expect(rows[1]['identifier']).to eq 'WRK001' expect(rows[1]['error']).to be_nil expect(rows[1]['title']).to include 'Test work' - expect(rows[1]['class']).to eq "Work" + expect(rows[1]['object type']).to eq "Work" end end diff --git a/spec/requests/export_request_spec.rb b/spec/requests/export_request_spec.rb new file mode 100644 index 00000000..d9d3da80 --- /dev/null +++ b/spec/requests/export_request_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +require 'rails_helper' + +RSpec.describe "/exports", type: :request do + let(:admin) { FactoryBot.create(:user, :admin) } + let(:export) { Export.new(user: admin) } + + before do + sign_in admin + end + + describe "GET /index" do + it "redirects to /jobs/index" do + get exports_path + expect(response).to redirect_to jobs_path + end + end + + describe "GET /new" do + it "renders a successful response" do + get new_export_path + expect(response).to render_template('exports/new') + end + end + + describe "GET /show" do + it "displays info for an export job" do + export.save! + get export_path export + expect(response).to render_template('exports/show') + end + end + + describe "POST /create" do + it "creates a new export job" do + expect { + post exports_path, params: { export: { identifiers: [] } } + }.to change(Export, :count).by(1) + end + + it "queues a new background job" do + ActiveJob::Base.queue_adapter = :test + expect { + post exports_path, params: { export: { identifiers: [] } } + } .to enqueue_job(BatchExportJob).with(Job.last).on_queue(:default) + end + + it "redirects to the submitted export show view" do + post exports_path, params: { export: { identifiers: [] } } + expect(response).to redirect_to Export.last + end + end +end diff --git a/spec/routing/exports_routing_spec.rb b/spec/routing/exports_routing_spec.rb new file mode 100644 index 00000000..040f2f09 --- /dev/null +++ b/spec/routing/exports_routing_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +require "rails_helper" + +RSpec.describe ExportsController, type: :routing do + describe "routing" do + it "routes to #index" do + expect(get: "/exports").to route_to("exports#index") + end + + it "routes to #new" do + expect(get: "/exports/new").to route_to("exports#new") + end + + it "routes to #show" do + expect(get: "/exports/1").to route_to("exports#show", id: "1") + end + + it "routes to #create" do + expect(post: "/exports").to route_to("exports#create") + end + + # Preflight jobs are not editable after creation + context 'invalid routes' do + it "does not route to #edit" do + expect(get: "/exports/1/edit").not_to be_routable + end + + it "does not route to #update via PUT" do + expect(put: "/exports/1").not_to be_routable + end + + it "does not route to #update via PATCH" do + expect(patch: "/exports/1").not_to be_routable + end + + it "does not route to #destroy" do + expect(delete: "/exports/1").not_to be_routable + end + end + end +end