[improvement] WhatsApp file uploading/sending and error handling (#2072)

TODO:

- [ ] Add specs for statuses/errors in
spec/requests/whats_app/three_sixty_dialog_webhook_spec.rb
- [x] refactor
app/adapters/whats_app_adapter/three_sixty_dialog_outbound/file.rb to
remove disabling rubocop

Specs for statuses/errors will be added in the PR that actually handles
statuses.

app/adapters/whats_app_adapter/three_sixty_dialog_outbound.rb

@@ -112,18 +112,16 @@ def send_message_template(recipient, message)
       end
 
       def send_message(recipient, message)
-        files = message.files
+        if message.files.present?
+          WhatsAppAdapter::ThreeSixtyDialogOutbound::File.perform_later(message_id: message.id)
+
+        else
 
-        if files.blank?
           WhatsAppAdapter::ThreeSixtyDialogOutbound::Text.perform_later(organization_id: message.organization.id,
                                                                         payload: text_payload(
                                                                           recipient, message.text
                                                                         ),
                                                                         message_id: message.id)
-        else
-          files.each do |_file|
-            WhatsAppAdapter::ThreeSixtyDialog::UploadFileJob.perform_later(message_id: message.id)
-          end
         end
       end
 

app/adapters/whats_app_adapter/three_sixty_dialog_outbound/file.rb

@@ -5,30 +5,52 @@ class ThreeSixtyDialogOutbound
     class File < ApplicationJob
       queue_as :default
 
-      def perform(message_id:, file_id:)
-        @message = Message.find(message_id)
-        organization = Organization.find(message.organization.id)
+      retry_on Net::HTTPServerError, wait: ->(executions) { executions * 3 } do |job, exception|
+        if job.executions == 5
+          exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: exception.code, message: exception.message)
+          context = { message_id: job.arguments.first[:message_id] }
+          ErrorNotifier.report(exception, context: context)
+        end
+      end
 
+      def perform(message_id:)
+        @message = Message.find(message_id)
         @recipient = message.recipient
-        @file_id = file_id
 
-        url = URI.parse("#{ENV.fetch('THREE_SIXTY_DIALOG_WHATS_APP_REST_API_ENDPOINT', 'https://stoplight.io/mocks/360dialog/360dialog-partner-api/24588693')}/messages")
-        headers = { 'D360-API-KEY' => organization.three_sixty_dialog_client_api_key, 'Content-Type' => 'application/json' }
-        request = Net::HTTP::Post.new(url.to_s, headers)
-        request.body = payload.to_json
-        response = Net::HTTP.start(url.host, url.port, use_ssl: true) do |http|
-          http.request(request)
-        end
-        handle_response(response)
-      rescue ActiveRecord::RecordNotFound => e
-        ErrorNotifier.report(e)
+        send_files
+        send_text_separately unless caption_it?
       end
 
       private
 
-      attr_reader :recipient, :file_id, :message
+      attr_reader :recipient, :message
+
+      def send_files
+        url = URI.parse("#{ENV.fetch('THREE_SIXTY_DIALOG_WHATS_APP_REST_API_ENDPOINT', 'https://stoplight.io/mocks/360dialog/360dialog-partner-api/24588693')}/messages")
+        headers = { 'D360-API-KEY' => message.organization.three_sixty_dialog_client_api_key, 'Content-Type' => 'application/json' }
+        request = Net::HTTP::Post.new(url, headers)
+
+        message.request.whats_app_external_file_ids.each do |file_id|
+          body = payload(file_id)
+          body[:image][:caption] = message.text if caption_it?
+
+          request.body = body.to_json
+          response = Net::HTTP.start(url.host, url.port, use_ssl: true) do |http|
+            http.request(request)
+          end
+          handle_response(response)
+        end
+      end
+
+      def send_text_separately
+        WhatsAppAdapter::ThreeSixtyDialogOutbound::Text.perform_later(
+          organization_id: message.organization_id,
+          payload: text_payload,
+          message_id: message.id
+        )
+      end
 
-      def payload
+      def payload(file_id)
         {
           messaging_product: 'whatsapp',
           recipient_type: 'individual',
@@ -40,16 +62,35 @@ def payload
         }
       end
 
+      def text_payload
+        {
+          messaging_product: 'whatsapp',
+          recipient_type: 'individual',
+          to: recipient.whats_app_phone_number.split('+').last,
+          type: 'text',
+          text: {
+            body: message.text
+          }
+        }
+      end
+
       def handle_response(response)
         case response
         when Net::HTTPSuccess
-          external_id = JSON.parse(response.body)['messages'].first['id']
-          message.update!(external_id: external_id)
-        when Net::HTTPClientError, Net::HTTPServerError
+          if caption_it?
+            external_id = JSON.parse(response.body)['messages'].first['id']
+            message.update!(external_id: external_id)
+          end
+        when Net::HTTPClientError
           exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: response.code, message: response.body)
-          ErrorNotifier.report(exception)
+          context = { message_id: message.id }
+          ErrorNotifier.report(exception, context: context)
         end
       end
+
+      def caption_it?
+        message.request.whats_app_external_file_ids.length.eql?(1) && message.text.length < 1024
+      end
     end
   end
 end

app/adapters/whats_app_adapter/three_sixty_dialog_outbound/text.rb

@@ -5,24 +5,31 @@ class ThreeSixtyDialogOutbound
     class Text < ApplicationJob
       queue_as :default
 
+      retry_on Net::HTTPServerError, wait: ->(executions) { executions * 3 } do |job, exception|
+        if job.executions == 5
+          exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: exception.code, message: exception.message)
+          context = { message_id: job.arguments.first[:message_id], recipient: job.payload[:to] }
+          ErrorNotifier.report(exception, context: context)
+        end
+      end
+
       def perform(organization_id:, payload:, message_id: nil)
-        organization = Organization.find(organization_id)
         @message = Message.find(message_id) if message_id
+        @payload = payload
+        organization = Organization.find(organization_id)
 
         url = URI.parse("#{ENV.fetch('THREE_SIXTY_DIALOG_WHATS_APP_REST_API_ENDPOINT', 'https://stoplight.io/mocks/360dialog/360dialog-partner-api/24588693')}/messages")
         headers = { 'D360-API-KEY' => organization.three_sixty_dialog_client_api_key, 'Content-Type' => 'application/json' }
-        request = Net::HTTP::Post.new(url.to_s, headers)
+        request = Net::HTTP::Post.new(url, headers)
 
         request.body = payload.to_json
         response = Net::HTTP.start(url.host, url.port, use_ssl: true) do |http|
           http.request(request)
         end
         handle_response(response)
-      rescue ActiveRecord::RecordNotFound => e
-        ErrorNotifier.report(e)
       end
 
-      attr_reader :message
+      attr_reader :message, :payload
 
       private
 
@@ -31,9 +38,10 @@ def handle_response(response)
         when Net::HTTPSuccess
           external_id = JSON.parse(response.body)['messages'].first['id']
           message&.update!(external_id: external_id)
-        when Net::HTTPClientError, Net::HTTPServerError
+        when Net::HTTPClientError
           exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: response.code, message: response.body)
-          ErrorNotifier.report(exception)
+          context = { message_id: message&.id, recipient: payload[:to] }
+          ErrorNotifier.report(exception, context: context)
         end
       end
     end

app/controllers/whats_app/three_sixty_dialog_webhook_controller.rb

@@ -7,9 +7,8 @@ class ThreeSixtyDialogWebhookController < ApplicationController
 
     def message
       head :ok
-      return if @components[:statuses].present? # TODO: Handle statuses
-
-      handle_error(@components[:errors].first) and return if @components[:errors].present?
+      handle_statuses and return if @components[:statuses].present? # TODO: Handle statuses
+      handle_errors(@components[:errors]) and return if @components[:errors].present?
 
       WhatsAppAdapter::ThreeSixtyDialog::ProcessWebhookJob.perform_later(organization_id: @organization.id, components: @components)
     end
@@ -25,8 +24,8 @@ def message_params
                     entry: [:id, {
                       changes: [:field, {
                         value: [:messaging_product,
-                                { metadata: %i[display_phone_number phone_number_id] },
-                                { contacts: [:wa_id, { profile: [:name] }],
+                                { metadata: %i[display_phone_number phone_number_id],
+                                  contacts: [:wa_id, { profile: [:name] }],
                                   messages: [:from, :id, :type, :timestamp,
                                              { text: [:body] }, { button: %i[payload text] },
                                              { image: %i[id mime_type sha256 caption] }, { voice: %i[id mime_type sha256] },
@@ -39,15 +38,26 @@ def message_params
                                              { context: %i[from id] }],
                                   statuses: [:id, :status, :timestamp, :expiration_timestamp, :recipient_id,
                                              { conversation: [:id, { origin: [:type] }] },
-                                             { pricing: %i[billable pricing_model category] }],
+                                             { pricing: %i[billable pricing_model category] },
+                                             { errors: [:code, :title, :message, :href, { error_data: [:details] }] }],
                                   errors: [:code, :title, :message, :href, { error_data: [:details] }] }]
+
                       }]
                     }])
     end
 
-    def handle_error(error)
-      exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: error[:code], message: error[:title])
-      ErrorNotifier.report(exception, context: { details: error[:error_data][:details] })
+    def handle_statuses
+      statuses = @components[:statuses]
+      statuses.each do |status|
+        handle_errors(status[:errors]) if status[:errors]
+      end
+    end
+
+    def handle_errors(errors)
+      errors.each do |error|
+        exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: error[:code], message: error[:title])
+        ErrorNotifier.report(exception, context: { details: error[:error_data][:details] })
+      end
     end
   end
 end

app/jobs/broadcast_request_job.rb

@@ -3,25 +3,40 @@
 class BroadcastRequestJob < ApplicationJob
   queue_as :broadcast_request
 
+  attr_reader :request, :recipients
+
   def perform(request_id)
-    request = Request.where(id: request_id).first
-    return unless request
+    @request = Request.find(request_id)
     return if request.broadcasted_at.present?
     return if request.planned? # rescheduled for future
 
-    request.organization.contributors.active.with_tags(request.tag_list).each do |contributor|
+    all_recipients = request.organization.contributors.active.with_tags(request.tag_list)
+    whats_app_recipients = all_recipients.with_whats_app
+    @recipients = all_recipients - whats_app_recipients
+
+    create_and_send_messages
+
+    WhatsAppAdapter::BroadcastMessagesJob.perform_later(request_id: request.id)
+
+    request.update(broadcasted_at: Time.current)
+  end
+
+  private
+
+  def create_and_send_messages
+    recipients.each do |contributor|
       message = Message.new(
         sender: request.user,
         recipient: contributor,
         text: request.personalized_text(contributor),
         request: request,
         broadcasted: true
       )
-      message.files = Request.attach_files(request.files) if request.files.attached?
+
+      message.files = Message::File.attach_files(request.files) if request.files.attached?
 
       message.save!
       message.send!
     end
-    request.update(broadcasted_at: Time.current)
   end
 end

app/jobs/whats_app_adapter/broadcast_messages_job.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module WhatsAppAdapter
+  class BroadcastMessagesJob < ApplicationJob
+    queue_as :broadcast_whats_app_messages
+
+    attr_reader :request, :recipients
+
+    def perform(request_id: request.id)
+      @request = Request.find(request_id)
+      @recipients = request.organization.contributors.active.with_tags(request.tag_list).with_whats_app
+
+      upload_files_to_meta if request.files.attached?
+      create_and_send_messages
+    end
+
+    private
+
+    def upload_files_to_meta
+      WhatsAppAdapter::ThreeSixtyDialog::UploadFileService.call(request_id: request.id)
+    end
+
+    def create_and_send_messages
+      recipients.each do |contributor|
+        message = Message.new(
+          sender: request.user,
+          recipient: contributor,
+          text: request.personalized_text(contributor),
+          request: request,
+          broadcasted: true
+        )
+
+        message.files = Message::File.attach_files(request.files) if request.files.attached?
+
+        message.save!
+        message.send!
+      end
+    end
+  end
+end

app/models/message/file.rb

@@ -27,4 +27,12 @@ def thumbnail
   def image_attachment?
     attachment.blob.content_type.match?(/image/)
   end
+
+  def self.attach_files(files)
+    files.map do |file|
+      message_file = Message::File.new
+      message_file.attachment.attach(file.blob)
+      message_file
+    end
+  end
 end

app/jobs/whats_app_adapter/three_sixty_dialog/upload_file_job.rb → app/services/whats_app_adapter/three_sixty_dialog/upload_file_service.rb

@@ -4,21 +4,19 @@
 
 module WhatsAppAdapter
   module ThreeSixtyDialog
-    class UploadFileJob < ApplicationJob
-      def perform(message_id:)
-        @message_id = message_id
-        message = Message.find_by(id: message_id)
-
-        request = message.request
-        organization = request.organization
+    class UploadFileService < ApplicationService
+      def initialize(request_id:)
+        @broadcasted_request = Request.find(request_id)
+        @base_uri = ENV.fetch('THREE_SIXTY_DIALOG_WHATS_APP_REST_API_ENDPOINT', 'https://stoplight.io/mocks/360dialog/360dialog-partner-api/24588693')
+      end
 
-        base_uri = ENV.fetch('THREE_SIXTY_DIALOG_WHATS_APP_REST_API_ENDPOINT', 'https://stoplight.io/mocks/360dialog/360dialog-partner-api/24588693')
+      def call
         url = URI.parse("#{base_uri}/media")
         headers = {
-          'D360-API-KEY' => organization.three_sixty_dialog_client_api_key
+          'D360-API-KEY' => broadcasted_request.organization.three_sixty_dialog_client_api_key
         }
 
-        request.files.each do |file|
+        broadcasted_request.files.each do |file|
           params = {
             'messaging_product' => 'whatsapp',
             'file' => UploadIO.new(ActiveStorage::Blob.service.path_for(file.blob.key), file.blob.content_type)
@@ -27,20 +25,20 @@ def perform(message_id:)
           response = Net::HTTP.start(url.host, url.port, use_ssl: true) do |http|
             http.request(request)
           end
-
           handle_response(response)
         end
+        broadcasted_request.save!
       end
 
       private
 
-      attr_reader :message_id
+      attr_reader :broadcasted_request, :base_uri
 
       def handle_response(response)
         case response
         when Net::HTTPSuccess
-          file_id = JSON.parse(response.body)['id']
-          WhatsAppAdapter::ThreeSixtyDialogOutbound::File.perform_later(message_id: message_id, file_id: file_id)
+          external_file_id = JSON.parse(response.body)['id']
+          broadcasted_request.whats_app_external_file_ids << external_file_id
         when Net::HTTPClientError, Net::HTTPServerError
           exception = WhatsAppAdapter::ThreeSixtyDialogError.new(error_code: response.code, message: response.body)
           ErrorNotifier.report(exception)

db/migrate/20241108101857_add_external_file_ids_to_requests.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddExternalFileIdsToRequests < ActiveRecord::Migration[6.1]
+  def change
+    add_column :requests, :external_file_ids, :string, array: true, default: []
+    add_index :requests, :external_file_ids, using: 'gin'
+  end
+end