Merge pull request #5852 from avalonmediasystem/sup_file_api

Add SupplementalFile ingest API

app/controllers/application_controller.rb

@@ -29,7 +29,7 @@ class ApplicationController < ActionController::Base
 
   helper_method :render_bookmarks_control?
 
-  around_action :handle_api_request, if: proc{|c| request.format.json? || request.format.atom? }
+  around_action :handle_api_request, if: proc{|c| request.format.json? || request.format.atom? || request.headers['Avalon-Api-Key'].present? }
   before_action :rewrite_v4_ids, if: proc{|c| request.method_symbol == :get && [params[:id], params[:content]].flatten.compact.any? { |i| i =~ /^[a-z]+:[0-9]+$/}}
   before_action :set_no_cache_headers, if: proc{|c| request.xhr? }
   prepend_before_action :remove_zero_width_chars

app/controllers/supplemental_files_controller.rb

@@ -14,6 +14,8 @@
 
 # frozen_string_literal: true
 class SupplementalFilesController < ApplicationController
+  include Rails::Pagination
+
   before_action :set_object
   before_action :authorize_object
 
@@ -30,19 +32,29 @@ class SupplementalFilesController < ApplicationController
     handle_error(message: exception.full_message, status: 404)
   end
 
-  def create
-    # FIXME: move filedata to permanent location
-    raise Avalon::BadRequest, "Missing required parameters" unless supplemental_file_params[:file]
+  def index
+    files = paginate SupplementalFile.where("parent_id = ?", @object.id)
+    render json: files.to_a.collect { |f| f.as_json }
+  end
 
-    @supplemental_file = SupplementalFile.new(label: supplemental_file_params[:label], tags: supplemental_file_params[:tags], parent_id: @object.id)
-    begin
-      @supplemental_file.attach_file(supplemental_file_params[:file])
-    rescue StandardError, LoadError => e
-      raise Avalon::SaveError, "File could not be attached: #{e.full_message}"
+  def create
+    if metadata_upload? && !attachment
+      raise Avalon::BadRequest, "Missing required Content-type headers" unless request.headers["Content-Type"] == 'application/json'
     end
+    raise Avalon::BadRequest, "Missing required parameters" unless validate_params
+
+    @supplemental_file = SupplementalFile.new(**metadata_from_params)
+
+    if attachment
+      begin
+        @supplemental_file.attach_file(attachment)
+      rescue StandardError, LoadError => e
+        raise Avalon::SaveError, "File could not be attached: #{e.full_message}"
+      end
 
-    # Raise errror if file wasn't attached
-    raise Avalon::SaveError, "File could not be attached." unless @supplemental_file.file.attached?
+      # Raise errror if file wasn't attached
+      raise Avalon::SaveError, "File could not be attached." unless @supplemental_file.file.attached?
+    end
 
     raise Avalon::SaveError, @supplemental_file.errors.full_messages unless @supplemental_file.save
 
@@ -52,43 +64,71 @@ def create
     flash[:success] = "Supplemental file successfully added."
 
     respond_to do |format|
-      format.html { redirect_to edit_structure_path }
-      format.json { head :created, location: object_supplemental_file_path }
+      format.html {
+        # This path is for uploading the binary file. We need to provide a JSON response
+        # for the case of someone uploading through a CLI.
+        if request.headers['Accept'] == 'application/json'
+          render json: { id: @supplemental_file.id }, status: :created
+        else
+          redirect_to edit_structure_path
+        end
+      }
+      # This path is for uploading the metadata payload.
+      format.json { render json: { id: @supplemental_file.id }, status: :created }
     end
   end
 
   def show
     find_supplemental_file
 
-    # Redirect or proxy the content
-    if Settings.supplemental_files.proxy
-      send_data @supplemental_file.file.download, filename: @supplemental_file.file.filename.to_s, type: @supplemental_file.file.content_type, disposition: 'attachment'
-    else
-      redirect_to rails_blob_path(@supplemental_file.file, disposition: "attachment")
+    respond_to do |format|
+      format.html { 
+        # Redirect or proxy the content
+        if Settings.supplemental_files.proxy
+          send_data @supplemental_file.file.download, filename: @supplemental_file.file.filename.to_s, type: @supplemental_file.file.content_type, disposition: 'attachment'
+        else
+          redirect_to rails_blob_path(@supplemental_file.file, disposition: "attachment")
+        end
+      }
+      format.json { render json: @supplemental_file.as_json }
     end
   end
 
-  # Update the label and tags of the supplemental file
   def update
-    raise Avalon::NotFound, "Cannot update the supplemental file: #{params[:id]} not found" unless SupplementalFile.exists? params[:id].to_s
-    @supplemental_file = SupplementalFile.find(params[:id])
-    raise Avalon::NotFound, "Cannot update the supplemental file: #{@supplemental_file.id} not found" unless @object.supplemental_files.any? { |f| f.id == @supplemental_file.id }
-    raise Avalon::BadRequest, "Updating file contents not allowed" if supplemental_file_params[:file].present?
+    if metadata_upload?
+      raise Avalon::BadRequest, "Incorrect request format. Use HTML if updating attached file." if attachment
+      raise Avalon::BadRequest, "Missing required Content-type headers" unless request.headers["Content-Type"] == 'application/json'
+    elsif request.headers['Avalon-Api-Key'].present?
+      raise Avalon::BadRequest, "Incorrect request format. Use JSON if updating metadata." unless attachment
+    end
+    raise Avalon::BadRequest, "Missing required parameters" unless validate_params
+
+    find_supplemental_file
+
+    edit_file_information if !attachment
+
+    @supplemental_file.attach_file(attachment) if attachment
 
-    edit_file_information
     raise Avalon::SaveError, @supplemental_file.errors.full_messages unless @supplemental_file.save
 
     flash[:success] = "Supplemental file successfully updated."
     respond_to do |format|
-      format.html { redirect_to edit_structure_path }
-      format.json { head :ok, location: object_supplemental_file_path }
+      format.html {
+        # This path is for uploading the binary file. We need to provide a JSON response
+        # for the case of someone uploading through a CLI.
+        if request.headers['Accept'] == 'application/json'
+          render json: { id: @supplemental_file.id }
+        else
+          redirect_to edit_structure_path
+        end
+      }
+      # This path is for uploading the metadata payload.
+      format.json { render json: { id: @supplemental_file.id }, status: :ok  }
     end
   end
 
   def destroy
-    raise Avalon::NotFound, "Cannot delete the supplemental file: #{params[:id]} not found" unless SupplementalFile.exists? params[:id].to_s
-    @supplemental_file = SupplementalFile.find(params[:id])
-    raise Avalon::NotFound, "Cannot delete the supplemental file: #{@supplemental_file.id} not found" unless @object.supplemental_files.any? { |f| f.id == @supplemental_file.id }
+    find_supplemental_file
 
     @object.supplemental_files -= [@supplemental_file]
     raise Avalon::SaveError, "An error occurred when deleting the supplemental file: #{@object.errors[:supplemental_files_json].full_messages}" unless @object.save
@@ -117,9 +157,34 @@ def set_object
       @object = fetch_object params[:master_file_id] || params[:media_object_id]
     end
 
+    def validate_params
+      attachment.present? || [:label, :language, :tags].any? { |v| supplemental_file_params[v].present? }
+    end
+
     def supplemental_file_params
       # TODO: Add parameters for minio and s3
-      params.fetch(:supplemental_file, {}).permit(:label, :language, :file, tags: [])
+      sup_file_params = params.fetch(:supplemental_file, {}).permit(:label, :language, :file, tags: [])
+      return sup_file_params unless metadata_upload?
+
+      meta_params = params[:metadata].present? ? JSON.parse(params[:metadata]).symbolize_keys : params
+
+      type = case meta_params[:type]
+             when 'caption'
+               'caption'
+             when 'transcript'
+               'transcript'
+             else
+               nil
+             end
+      treat_as_transcript = 'transcript' if meta_params[:treat_as_transcript] == true
+      machine_generated = 'machine_generated' if meta_params[:machine_generated] == true
+
+      sup_file_params[:label] ||= meta_params[:label].presence
+      sup_file_params[:language] ||= meta_params[:language].presence
+      # The uniq is to prevent multiple instances of 'transcript' tag if an update is performed with
+      # `{ type: transcript, treat_as_transcript: 1}`
+      sup_file_params[:tags] ||= [type, treat_as_transcript, machine_generated].compact.uniq
+      sup_file_params
     end
 
     def find_supplemental_file
@@ -133,7 +198,7 @@ def find_supplemental_file
 
 
     def handle_error(message:, status:)
-      if request.format == :json
+      if request.format == :json || request.headers['Avalon-Api-Key'].present?
         render json: { errors: message }, status: status
       else
         flash[:error] = message
@@ -151,6 +216,22 @@ def edit_structure_path
     end
 
     def edit_file_information
+      update_tags
+
+      @supplemental_file.label = supplemental_file_params[:label]
+      return unless supplemental_file_params[:language].present?
+      @supplemental_file.language = LanguageTerm.find(supplemental_file_params[:language]).code
+    end
+
+    def update_tags
+      # The edit page only provides supplemental_file_params[:tags] on object creation.
+      # Thus, we need to provide individual handling for both updates triggered by page
+      # actions and updates through the JSON api.
+      if request.format == 'json'
+        @supplemental_file.tags = supplemental_file_params[:tags].presence
+        return
+      end
+
       file_params = [ 
         { param: "machine_generated_#{params[:id]}".to_sym, tag: "machine_generated", method: :machine_generated? },
         { param: "treat_as_transcript_#{params[:id]}".to_sym, tag: "transcript", method: :caption_transcript? }
@@ -166,9 +247,23 @@ def edit_file_information
           @supplemental_file.tags -= [tag]
         end
       end
-      @supplemental_file.label = supplemental_file_params[:label]
-      return unless supplemental_file_params[:language].present?
-      @supplemental_file.language = LanguageTerm.find(supplemental_file_params[:language]).code
+    end
+
+    def metadata_from_params
+      {
+        label: supplemental_file_params[:label],
+        tags: supplemental_file_params[:tags],
+        language: supplemental_file_params[:language].present? ? LanguageTerm.find(supplemental_file_params[:language]).code : Settings.caption_default.language,
+        parent_id: @object.id
+      }.compact
+    end
+
+    def metadata_upload?
+      params[:format] == 'json'
+    end
+
+    def attachment
+      params[:file] || supplemental_file_params[:file]
     end
 
     def object_supplemental_file_path

app/models/supplemental_file.rb

@@ -22,26 +22,22 @@ class SupplementalFile < ApplicationRecord
   # TODO: the empty tag should represent a generic supplemental file
   validates :tags, array_inclusion: ['transcript', 'caption', 'machine_generated', '', nil]
   validates :language, inclusion: { in: LanguageTerm.map.keys }
-  validate  :validate_file_type, if: :caption?
   validates :parent_id, presence: true
+  validate  :validate_file_type, if: :caption?
 
   serialize :tags, Array
 
   # Need to prepend so this runs before the callback added by `has_one_attached` above
   # See https://github.com/rails/rails/issues/37304
-  after_create_commit :update_index, prepend: true
-  after_update :update_index
-
-  def validate_file_type
-    errors.add(:file_type, "Uploaded file is not a recognized captions file") unless ['text/vtt', 'text/srt'].include? file.content_type
-  end
+  after_create_commit :index_file, prepend: true
+  after_update_commit :update_index, prepend: true
 
   def attach_file(new_file)
     file.attach(new_file)
     extension = File.extname(new_file.original_filename)
     self.file.content_type = Mime::Type.lookup_by_extension(extension.slice(1..-1)).to_s if extension == '.srt'
     self.label = file.filename.to_s if label.blank?
-    self.language = Settings.caption_default.language
+    self.language ||= Settings.caption_default.language
   end
 
   def mime_type
@@ -64,6 +60,25 @@ def caption_transcript?
     tags.include?('caption') && tags.include?('transcript')
   end
 
+  def as_json(options={})
+    type = if tags.include?('caption')
+             'caption'
+           elsif tags.include?('transcript')
+             'transcript'
+           else
+             'generic'
+           end
+
+    {
+      id: id,
+      type: type,
+      label: label,
+      language: LanguageTerm.find(language).text,
+      treat_as_transcript: caption_transcript? ? true : false,
+      machine_generated: machine_generated? ? true : false
+    }.compact
+  end
+
   # Adapted from https://github.com/opencoconut/webvtt-ruby/blob/e07d59220260fce33ba5a0c3b355e3ae88b99457/lib/webvtt/parser.rb#L11-L30
   def self.convert_from_srt(srt)
     # normalize timestamps in srt
@@ -80,9 +95,13 @@ def self.convert_from_srt(srt)
     "WEBVTT\n\n#{conversion}".strip
   end
 
+  # We need to use both after_create_commit and after_update_commit to update the index properly in both cases. 
+  # However, they cannot call the same method name or only the last defined callback will take effect.
+  # https://guides.rubyonrails.org/active_record_callbacks.html#aliases-for-after-commit
   def update_index
-    ActiveFedora::SolrService.add(to_solr, softCommit: true)
+    ActiveFedora::SolrService.add(to_solr, softCommit: true) if file.present?
   end
+  alias index_file update_index
 
   # Creates a solr document hash for the {#object}
   # @return [Hash] the solr document
@@ -111,6 +130,11 @@ def segment_transcript transcript
 
   private
 
+  def validate_file_type
+    return unless file.present?
+    errors.add(:file_type, "Uploaded file is not a recognized captions file") unless ['text/vtt', 'text/srt'].include?(file.content_type)
+  end
+
   def c_time
     created_at&.to_datetime || DateTime.now
   end

config/routes.rb

@@ -139,7 +139,9 @@
     end
 
     # Supplemental Files
-    resources :supplemental_files, except: [:new, :index, :edit]
+    resources :supplemental_files, except: [:new, :index, :edit] do
+      get :index, constraints: { format: 'json' }, on: :collection
+    end
   end
 
   resources :master_files, except: [:new, :index] do
@@ -173,6 +175,7 @@
         get 'captions'
         get 'transcripts', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
       end
+      get :index, constraints: { format: 'json' }, on: :collection
     end
   end