Add forecasts + make imports more robust

Gemfile

@@ -26,6 +26,7 @@ gem "polaris_view_components"
 gem "chartkick"
 gem "groupdate"
 gem "convenient_grouper"
+gem "prophet-rb"
 
 # Importing
 gem "csvreader"

Gemfile.lock

@@ -108,6 +108,7 @@ GEM
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
     chartkick (5.0.4)
+    cmdstan (0.2.2)
     coderay (1.1.3)
     concurrent-ruby (1.2.2)
     connection_pool (2.4.1)
@@ -190,6 +191,7 @@ GEM
       racc (~> 1.4)
     nokogiri (1.15.4-x86_64-linux)
       racc (~> 1.4)
+    numo-narray (0.9.2.1)
     orm_adapter (0.5.0)
     parallel (1.23.0)
     parser (3.2.2.3)
@@ -199,6 +201,10 @@ GEM
     polaris_view_components (1.1.0)
       rails (>= 5.0.0)
       view_component (>= 3.0.0, < 4.0.0)
+    prophet-rb (0.5.0)
+      cmdstan (>= 0.2)
+      numo-narray (>= 0.9.1.7)
+      rover-df
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -256,6 +262,8 @@ GEM
       actionpack (>= 5.2)
       railties (>= 5.2)
     rexml (3.2.6)
+    rover-df (0.3.4)
+      numo-narray (>= 0.9.1.9)
     rubocop (1.56.1)
       base64 (~> 0.1.1)
       json (~> 2.3)
@@ -379,6 +387,7 @@ DEPENDENCIES
   letter_opener
   pg (~> 1.1)
   polaris_view_components
+  prophet-rb
   pry-rails
   puma (~> 6.0)
   rack-timeout

app/controllers/imports/globes_controller.rb

@@ -2,9 +2,19 @@ class Imports::GlobesController < ApplicationController
   before_action :authenticate_user!
 
   def show
-    payments = current_user.payments.pluck(:shop_country, :charge_type).last(30)
-    globe_data = payments.map { |c| {countryCode: c[0], reverse: c[1] == "refund"} }
+    @import = current_user.imports.find(params[:import_id])
 
     render json: globe_data
   end
+
+  private
+
+  def globe_data
+    payments = @import.payments.pluck(:shop_country, :charge_type).last(80)
+    globe_data = []
+    payments.each do |c|
+      globe_data << {countryCode: c[0], reverse: c[1] == "refund"}
+    end
+    globe_data
+  end
 end

app/controllers/imports_controller.rb

@@ -15,9 +15,12 @@ def new
   end
 
   def create
-    @import = current_user.imports.new(source: Import.sources[:csv_file], **import_params)
-    @import.user.count_usage_charges_as_recurring = import_params.dig(:user_attributes, :count_usage_charges_as_recurring)
+    @import = current_user.imports.new(
+      source: Import.sources[:csv_file],
+      **import_params.except(:user_attributes)
+    )
     if @import.save
+      save_user_attributes
       redirect_to @import, notice: "Import successfully created."
     else
       flash.now[:alert] = "Import failed to create."
@@ -40,7 +43,13 @@ def import_params
     params.require(:import).permit(
       :import_type,
       :payouts_file,
-      user_attributes: [:count_usage_charges_as_recurring]
+      user_attributes: [:id, :count_usage_charges_as_recurring]
+    )
+  end
+
+  def save_user_attributes
+    current_user.update(
+      count_usage_charges_as_recurring: import_params.dig(:user_attributes, :count_usage_charges_as_recurring)
     )
   end
 

app/controllers/partner_api_credentials_controller.rb

@@ -15,9 +15,11 @@ def edit
   end
 
   def create
-    @partner_api_credential = current_user.build_partner_api_credential(partner_api_credential_params)
-
+    @partner_api_credential = current_user.build_partner_api_credential(
+      partner_api_credential_params.except(:user_attributes)
+    )
     if @partner_api_credential.save
+      save_user_attributes
       redirect_to edit_partner_api_credential_path(@partner_api_credential), notice: "Partner api credential was successfully created."
     else
       render :new, status: :unprocessable_entity, notice: "Partner api credential was not created."
@@ -51,12 +53,17 @@ def add_status_messsage_error
     @partner_api_credential.errors.add(:base, @partner_api_credential.status_message)
   end
 
-  # Only allow a list of trusted parameters through.
+  def save_user_attributes
+    current_user.update(
+      count_usage_charges_as_recurring: partner_api_credential_params.dig(:user_attributes, :count_usage_charges_as_recurring)
+    )
+  end
+
   def partner_api_credential_params
     params.require(:partner_api_credential).permit(
       :access_token,
       :organization_id,
-      user_attributes: [:count_usage_charges_as_recurring]
+      user_attributes: [:id, :count_usage_charges_as_recurring]
     )
   end
 end

app/controllers/users_controller.rb

@@ -0,0 +1,17 @@
+class UsersController < ApplicationController
+  before_action :authenticate_user!
+
+  def update
+    if current_user.update(user_params)
+      redirect_to request.referrer, status: :see_other
+    else
+      redirect_to request.referrer, alert: "Error saving user!", status: :see_other
+    end
+  end
+
+  private
+
+  def user_params
+    params.require(:user).permit(:show_forecasts)
+  end
+end

app/helpers/metrics_helper.rb

@@ -80,14 +80,15 @@ def metrics_chart_options
       download: true,
       library: {
         pointSize: 6,
+        isStacked: false,
         backgroundColor: "transparent",
         animation: {
           startup: true,
           duration: 600,
           easing: "inAndOut"
         },
         lineWidth: 3,
-        colors: ["#5912D5", "rgba(89, 18, 213, 0.2)"],
+        colors: ["#5912D5", "#5912D5"],
         explorer: {
           keepInBounds: true,
           axis: "horizontal",
@@ -101,10 +102,24 @@ def metrics_chart_options
             color: "#F1F2F4"
           }
         },
+        timeline: {
+          tooltipDateFormat: "MMM d, yyyy"
+        },
+        hAxis: {
+          format: "MMM d, y"
+        },
         chartArea: {
           width: "100%",
           height: "80%",
           left: "5%"
+        },
+        focusTarget: "category",
+        series: {
+          "1": {
+            visibleInLegend: false,
+            lineWidth: 0,
+            areaOpacity: 0
+          }
         }
       }
     }

app/javascript/controllers/form_controller.js

@@ -4,7 +4,7 @@ import DirtyForm from '../libraries/dirty-form'
 export default class extends Controller {
   static targets = ['submitButton']
   static values = {
-    showSaveBar: { type: Boolean, default: true },
+    showSaveBar: { type: Boolean, default: false },
     submitButtonTarget: String
   }
 

app/javascript/controllers/globe_controller.js

@@ -32,7 +32,7 @@ const ARC_ALTITUDES = [0.5, 0.55, 0.55, 0.6, 0.6];
 const ARC_DASH_LENGTHS = [1.5, 1.8, 1.9, 1.9, 2, 2.2];
 const ARC_SPEEDS = [0.6, 0.65, 0.75, 0.95, 1];
 
-const ARCS_AT_ONCE = 5;
+const ARCS_AT_ONCE = 8;
 const ARC_EMIT_DELAY = 300;
 
 export default class extends Controller {
@@ -295,7 +295,7 @@ export default class extends Controller {
     controls.rotateSpeed = 0.2;
     controls.zoomSpeed = 0.2;
     controls.autoRotate = true;
-    controls.autoRotateSpeed = 0.35;
+    controls.autoRotateSpeed = 0.25;
   }
 
   // SETUP CAMERA

app/jobs/calculate_metrics_job.rb

@@ -5,7 +5,7 @@ class CalculateMetricsJob < ApplicationJob
   def perform(import:)
     import.calculate
   rescue => e
-    import&.failed!
+    import&.fail
     raise e
   end
 end

app/jobs/import_payments_job.rb

@@ -5,7 +5,7 @@ class ImportPaymentsJob < ApplicationJob
   def perform(import:)
     import.import
   rescue => e
-    import&.failed!
+    import&.fail
     raise e
   end
 end

app/models/import.rb

@@ -43,7 +43,7 @@ def schedule
 
   def import
     importing!
-    Import::Payments.new(import: self).import!
+    Import::PaymentsProcessor.new(import: self).import!
     imported
   end
 
@@ -53,14 +53,42 @@ def imported
 
   def calculate
     calculating!
-    Import::Metrics.new(import: self).calculate!
+    Import::MetricsProcessor.new(import: self).calculate!
     completed!
   end
 
+  def fail
+    failed!
+    payments.delete_all
+    metrics.delete_all
+  end
+
   def source_adaptor
     csv_file_source? ? Import::Adaptor::CsvFile : Import::Adaptor::ShopifyPaymentsApi
   end
 
+  def import_payments_after_date
+    max_allowed_ago = source_adaptor.const_get(:MAX_HISTORY).ago
+    if user.newest_metric_date.to_i < max_allowed_ago.to_i
+      max_allowed_ago
+    else
+      user.newest_metric_date
+    end
+  end
+
+  def import_metrics_after_date
+    if user.newest_metric_date.present?
+      user.newest_metric_date + 1.day
+    else
+      user.payments.minimum("payment_date")
+    end
+  end
+
+  def import_metrics_before_date
+    # Don't include the latest day, because it may not be complete
+    user.payments.maximum(:payment_date) - 1.day
+  end
+
   private
 
   def broadcast_details_update

app/models/import/adaptor/csv_file.rb

@@ -2,10 +2,10 @@
 require "csvreader"
 
 class Import::Adaptor::CsvFile
-  BATCH_SIZE = 500
+  BATCH_SIZE = 1000
+  MAX_HISTORY = 4.years
 
   CSV_READER_OPTIONS = {
-    converters: :all,
     header_converters: :symbol,
     encoding: "UTF-8"
   }.freeze
@@ -57,11 +57,12 @@ class Import::Adaptor::CsvFile
     ]
   }.freeze
 
-  def initialize(import:, created_at_min:)
+  def initialize(import:, import_payments_after_date:)
     @import = import
-    @created_at_min = created_at_min
+    @import_payments_after_date = import_payments_after_date
 
     @temp_files = {}
+    @prepared_csv_file = prepared_csv_file
   end
 
   def fetch_payments
@@ -75,9 +76,9 @@ def batch_size
   private
 
   def stream_payments(main_enum)
-    CsvHashReader.foreach(prepared_csv_file, **CSV_READER_OPTIONS) do |csv_row|
+    CsvHashReader.foreach(@prepared_csv_file, **CSV_READER_OPTIONS) do |csv_row|
       parsed_row = parse(csv_row)
-      break if parsed_row[:payment_date] <= @created_at_min
+      break if parsed_row[:payment_date] <= @import_payments_after_date
 
       main_enum.yield parsed_row
     end
@@ -122,6 +123,8 @@ def shop_country(csv_row)
   end
 
   def prepared_csv_file
+    return @prepared_csv_file if @prepared_csv_file
+
     file = @import.payouts_file
     if zipped?(file.content_type)
       extracted_zip_file(ActiveStorage::Blob.service.path_for(file.key))

app/models/import/adaptor/shopify_payments_api.rb

@@ -5,8 +5,9 @@
 class Import::Adaptor::ShopifyPaymentsApi
   include ShopifyPartnerAPI
 
-  THROTTLE_MIN_TIME_PER_CALL = 0.3
   BATCH_SIZE = 100
+  MAX_HISTORY = 7.days
+  THROTTLE_MIN_TIME_PER_CALL = 0.3.seconds
 
   API_REVENUE_TYPES = {
     "recurring_revenue" => [
@@ -36,11 +37,11 @@ class Import::Adaptor::ShopifyPaymentsApi
     ]
   }.freeze
 
-  def initialize(import:, created_at_min:)
+  def initialize(import:, import_payments_after_date:)
     @import = import
-    @created_at_min = created_at_min.strftime("%Y-%m-%dT%H:%M:%S.%L%z")
+    @import_payments_after_date = import_payments_after_date.strftime("%Y-%m-%dT%H:%M:%S.%L%z")
 
-    @context = @import.partner_api_credential.context
+    @context = import.partner_api_credential.context
     @cursor = ""
     @throttle_start_time = Time.zone.now
   end
@@ -61,7 +62,7 @@ def stream_payments(main_enum)
     while has_next_page
       @throttle_start_time = throttle(@throttle_start_time)
 
-      results = fetch_from_api(@cursor, @created_at_min)
+      results = fetch_from_api(@cursor)
       break if results.data.nil?
 
       transactions = results.data.transactions.edges
@@ -74,10 +75,10 @@ def stream_payments(main_enum)
     end
   end
 
-  def fetch_from_api
+  def fetch_from_api(cursor)
     results = ShopifyPartnerAPI.client.query(
       Graphql::TransactionsQuery,
-      variables: {createdAtMin: @created_at_min, cursor: @cursor, first: batch_size},
+      variables: {cursor: cursor, createdAtMin: @import_payments_after_date, first: batch_size},
       context: @context
     )
     handle_error(results.errors) if results.errors.any?