Move from delayed job to solid and use Active Job

Dockerfile

@@ -60,8 +60,6 @@ FROM base
 COPY --from=build "${BUNDLE_PATH}" "${BUNDLE_PATH}"
 COPY --from=build --exclude=tmp/cache/* --exclude=vendor/bundle /rails /rails
 
-RUN ln -nfs /dev/stdout log/delayed_job.log
-
 # Run and own only the runtime files as a non-root user for security
 RUN groupadd --system --gid 1000 rails && \
     useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \

Gemfile

@@ -60,6 +60,9 @@ end
 # Our stuff
 ###################
 
+gem 'solid_queue' # New queue system coming to Rails 8
+gem "mission_control-jobs" # Admin backend for jobs
+
 # Needs to be as early as possible to do it's job
 gem 'dotenv-rails', require: 'dotenv/load' # , groups: [:development, :test] # Load env variables in dev
 
@@ -72,7 +75,6 @@ gem 'oai' # OAI-PMH
 gem 'rexml' # OAI needs it https://github.com/code4lib/ruby-oai/issues/68
 
 # Analytics and instrumentation
-gem 'sentry-delayed_job'
 gem 'sentry-rails'
 gem 'sentry-ruby'
 
@@ -89,13 +91,10 @@ gem 'paper_trail' # Keep an audit trail of all the changes
 # Background processing
 gem 'aws-sdk-rails' # Send emails via SES
 gem 'aws-sdk-s3' # Talk to the catalog
-gem 'daemons' # Needed by delayed_job
-gem 'delayed_job_active_record' # Delay jobs and queue them in the database
 
 # Frameworks
 gem 'activeadmin'
 gem 'country_select', '~> 8.0' # 9.0 breaks active admin
-gem 'delayed-web'
 gem 'graphiql-rails', '1.8.0' # https://github.com/rmosolgo/graphiql-rails/issues/106
 gem 'graphql'
 

Gemfile.lock

@@ -170,7 +170,6 @@ GEM
       countries (~> 5.0)
     crass (1.0.6)
     csv (3.3.0)
-    daemons (1.4.1)
     database_cleaner (2.0.2)
       database_cleaner-active_record (>= 2, < 3)
     database_cleaner-active_record (2.2.0)
@@ -181,13 +180,6 @@ GEM
     debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    delayed-web (0.4.9)
-      rails (>= 3.2.13)
-    delayed_job (4.1.12)
-      activesupport (>= 3.0, < 8.0)
-    delayed_job_active_record (4.1.8)
-      activerecord (>= 3.0, < 8.0)
-      delayed_job (>= 3.0, < 5)
     devise (4.9.4)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
@@ -339,6 +331,12 @@ GEM
     method_source (1.1.0)
     mini_mime (1.1.5)
     minitest (5.25.0)
+    mission_control-jobs (0.3.1)
+      importmap-rails
+      irb (~> 1.13)
+      rails (>= 7.1)
+      stimulus-rails
+      turbo-rails
     msgpack (1.7.2)
     multi_json (1.15.0)
     mysql2 (0.5.6)
@@ -560,9 +558,6 @@ GEM
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
-    sentry-delayed_job (5.19.0)
-      delayed_job (>= 4.0)
-      sentry-ruby (~> 5.19.0)
     sentry-rails (5.19.0)
       railties (>= 5.0)
       sentry-ruby (~> 5.19.0)
@@ -587,6 +582,13 @@ GEM
       thor (~> 1.0)
       tilt (~> 2.0)
       yard (~> 0.9, >= 0.9.24)
+    solid_queue (0.8.2)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
+      thor (~> 1.3.1)
     sorbet (0.5.11520)
       sorbet-static (= 0.5.11520)
     sorbet-runtime (0.5.11520)
@@ -673,11 +675,8 @@ DEPENDENCIES
   cancancan
   capybara
   country_select (~> 8.0)
-  daemons
   database_cleaner
   debug
-  delayed-web
-  delayed_job_active_record
   devise
   doorkeeper
   dotenv-rails
@@ -694,6 +693,7 @@ DEPENDENCIES
   json_schemer
   kaminari
   letter_opener
+  mission_control-jobs
   mysql2 (~> 0.5)
   nilify_blanks
   oai
@@ -722,10 +722,10 @@ DEPENDENCIES
   searchjoy
   searchkick
   selenium-webdriver
-  sentry-delayed_job
   sentry-rails
   sentry-ruby
   solargraph
+  solid_queue
   sorbet
   sprockets-rails
   stimulus-rails

TODO.md

@@ -1,6 +1,7 @@
 # TODO
 
 * Set @admin_rocrate false
+* Do solidQ/activeJob errors get caught by sentry?
 
 ## Rails 7.1
 Can we get rid of sassc??
@@ -34,7 +35,6 @@ If we give tokens to more than paragest we need to rethink this
 * Remove to_csv in collections, and replace with standard CSV. Similar to the service used for items
 * Move to_rif in collections controller to a haml template
 * Should we use authenticate_users on dashboard or should cancan just do it?
-* Should we use active job and let it use delayed_job
 * DB is all latin1 can we move to utf8mb4
 * SHoudl we add has_paper_trail to party_identifier
 * set set_paper_trail_whodunnit - check live db to see if it's empty for users

app/controllers/items_controller.rb

@@ -162,9 +162,9 @@ def bulk_update
     accessible_items = Item.accessible_by(current_ability)
                            .where(id: params[:item_ids].split)
                            .pluck(:id)
-    BulkUpdateItemsService.new(item_ids: accessible_items,
+    BulkUpdateItemsJob.perform_later(item_ids: accessible_items,
                                current_user_email: current_user.try(:email),
-                               updates: item_params).delay.update_items
+                               updates: item_params)
 
     flash[:notice] = "Items will be updated shortly, you'll be notified once it's completed"
     redirect_to bulk_update_items_path + "?#{params[:original_search_params]}"
@@ -354,7 +354,7 @@ def find_item
 
   def stream_csv(search_type)
     export_all = params[:export_all] || false
-    downloader = CsvDownloader.new(search_type, @params, current_user)
+    downloader = CsvDownloaderService.new(search_type, @params, current_user)
 
     # TODO: fix CSV stream for builder method
 
@@ -372,8 +372,8 @@ def stream_csv(search_type)
       return
     end
 
-    # otherwise use delayed_job to email a CSV
-    downloader.delay.email
+    # otherwise use actove job to email a CSV
+    downloader.email
 
     flash[:notice] = 'Your CSV file was too large to download directly. It will be generated and sent to you via email.'
     redirect_back fallback_location: root_path

app/services/bulk_update_items_service.rb → app/jobs/bulk_update_items_job.rb

@@ -1,11 +1,17 @@
-class BulkUpdateItemsService
-  def initialize(item_ids:, updates:, current_user_email:)
+class BulkUpdateItemsJob < ApplicationJob
+  queue_as :default
+
+  def perform(item_ids:, updates:, current_user_email:)
     @item_ids = item_ids
     @current_user_email = current_user_email
     @start_time = Time.current
+
     process_updates(updates)
+    update_items
   end
 
+  private
+
   def update_items
     Rails.logger.info { "#{DateTime.now} BEFORE Bulk update with #{items.size} items" }
     @failed_items = []

app/jobs/catalog_metadata_job.rb

@@ -0,0 +1,19 @@
+class CatalogMetadataJob < ApplicationJob
+  queue_as :default
+
+  def perform(data, is_item)
+    local_data = { data:, is_item:, admin_rocrate: true }
+
+    rocrate = Api::V1::OniController.render :object_meta, assigns: local_data
+
+    filename = 'ro-crate-metadata.json'
+
+    if is_item
+      # data = data.includes(:content_languages, :subject_languages, item_agents: %i[agent_role user])
+      Nabu::Catalog.instance.upload_item_admin(data, filename, rocrate, 'application/json')
+    else
+      # data = data.includes(items: [:admins, :users]) # => { :content_languages, :subject_languages, item_agents: %i[agent_role user] } })
+      Nabu::Catalog.instance.upload_collection_admin(data, filename, rocrate, 'application/json')
+    end
+  end
+end

app/mailers/csv_download_mailer.rb

@@ -11,5 +11,7 @@ def csv_download_email(email, user_name, total, start_time, filename, path)
     puts "Sending CSV download to #{@user_name} [#{@email}] with attachment #{@filename} containing #{@total} results"
 
     mail(to: email, subject: "Nabu - CSV export started on #{start_time.strftime('%a %e %b %Y %l:%M %P')} has been completed")
+
+    File.delete(path)
   end
 end

app/models/collection.rb

@@ -171,7 +171,7 @@ def full_grant_identifier(grant)
   end
 
   def update_catalog_metadata
-    CatalogMetadataService.new(self, false).delay.save_file
+    CatalogMetadataJob.perform_later(self, false)
   end
 
   searchkick geo_shape: [:bounds], word_start: [:identifier]

app/models/item.rb

@@ -631,7 +631,7 @@ def bulk_deleteable
   end
 
   def update_catalog_metadata
-    CatalogMetadataService.new(self, true).delay.save_file
+    CatalogMetadataJob.perform_later(self, true)
   end
 
   def center_coordinate

app/services/csv_downloader.rb → app/services/csv_downloader_service.rb

@@ -1,6 +1,6 @@
 require 'zip'
 
-class CsvDownloader
+class CsvDownloaderService
   INCLUDED_CSV_FIELDS = %i[full_identifier title external description url collector_sortname operator_name csv_item_agents
                            csv_filenames csv_mimetypes csv_fps_values csv_samplerates csv_channel_counts
                            university_name language dialect csv_subject_languages csv_content_languages csv_countries region csv_data_categories csv_data_types
@@ -21,7 +21,20 @@ def initialize(search_type, params, current_user)
 
   attr_reader :params, :current_user
 
+  def create_csv(search, csv)
+    search.each { |r| csv << INCLUDED_CSV_FIELDS.map { |f| r.public_send(f) } }
+
+    # if the user requested all results, iterate over the remaining pages
+    while @params[:export_all] && search.next_page
+      @params.merge!(page: search.next_page)
+      search = @search_type == :basic ? build_basic_search : build_advanced_search
+      search.each { |r| csv << INCLUDED_CSV_FIELDS.map { |f| r.public_send(f) } }
+    end
+  end
+
   def email
+    return unless @current_user.email.present?
+
     generation_start = DateTime.now
     search = @search_type == :basic ? build_basic_search : build_advanced_search
 
@@ -30,13 +43,7 @@ def email
     path = Rails.root.join('tmp', "nabu_items_#{Time.zone.today}.csv").to_s
 
     CSV.open(path, 'wb', **CSV_OPTIONS) do |csv|
-      search.each { |r| csv << INCLUDED_CSV_FIELDS.map { |f| r.public_send(f) } }
-      # if the user requested all results, iterate over the remaining pages
-      while @params[:export_all] && search.next_page
-        @params.merge!(page: search.next_page)
-        search = @search_type == :basic ? build_basic_search : build_advanced_search
-        search.each { |r| csv << INCLUDED_CSV_FIELDS.map { |f| r.public_send(f) } }
-      end
+      create_csv(search, csv)
     end
 
     total = @params[:export_all] ? search.total_count : (@params[:per_page] || 10)
@@ -51,39 +58,26 @@ def email
       zipfile.add(File.basename(path), path)
     end
 
-    if @current_user.email.present?
-      CsvDownloadMailer.csv_download_email(
-        @current_user.email,
-        # default just first name, but fall back to last in case of only one name
-        @current_user.first_name || @current_user.last_name,
-        total,
-        @csv_requested_time.in_time_zone('Australia/Sydney'),
-        filename,
-        zip_path
-      ).deliver
-    end
+    CsvDownloadMailer.csv_download_email(
+      @current_user.email,
+      # default just first name, but fall back to last in case of only one name
+      @current_user.first_name || @current_user.last_name,
+      total,
+      @csv_requested_time.in_time_zone('Australia/Sydney'),
+      filename,
+      zip_path
+    ).deliver_later
 
     File.delete(path)
-    File.delete(zip_path)
   end
 
-  def stream(orig_search)
+  def stream(search)
     filename = "nabu_items_#{Time.zone.today}.csv"
 
-    search = orig_search
-
     # use enumerator to customise streaming the response
     streamed_csv = lambda { |output|
-      # wrap the IO output so that CSV pushes writes directly into it
       csv = CSV.new(output, **CSV_OPTIONS)
-      search.each { |r| csv << INCLUDED_CSV_FIELDS.map { |f| r.public_send(f) } }
-
-      # if the user requested all results, iterate over the remaining pages
-      while @params[:export_all] && search.next_page
-        @params.merge!(page: search.next_page)
-        search = @search_type == :basic ? build_basic_search : build_advanced_search
-        search.each { |r| csv << INCLUDED_CSV_FIELDS.map { |f| r.public_send(f) } }
-      end
+      create_csv(search, csv)
     }
 
     [filename, streamed_csv]

bin/jobs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)

cdk/lib/app-stack.ts

@@ -331,7 +331,7 @@ export class AppStack extends cdk.Stack {
       ...commonAppImageOptions,
       memoryLimitMiB: 1024,
       logging: ecs.LogDrivers.awsLogs({ streamPrefix: 'JobsService' }),
-      command: ['bin/delayed_job', 'run'],
+      command: ['bin/jobs'],
     });
     jobsTaskDefinition.addToTaskRolePolicy(
       new iam.PolicyStatement({

config/application.rb

@@ -33,8 +33,6 @@ class Application < Rails::Application
 
     config.viewer_url = '/viewer'
 
-    config.assets.precompile << 'delayed/web/application.css'
-
     config.catalog_bucket = ENV.fetch('NABU_CATALOG_BUCKET')
     throw 'Must set NABU_CATALOG_BUCKET' unless config.catalog_bucket
   end