Merge pull request #943 from Vizzuality/hotfix-downloads-bug

Hotfix downloads bug
Vizzuality · Aug 8, 2019 · a1d3409 · a1d3409
2 parents 01bd006 + 5e42382
commit a1d3409
Show file tree

Hide file tree

Showing 11 changed files with 280 additions and 1,302 deletions.
diff --git a/app/models/api/v3/readonly/download_flow.rb b/app/models/api/v3/readonly/download_flow.rb
@@ -2,7 +2,6 @@
 #
 # Table name: download_flows
 #
-#  id             :integer          primary key
 #  context_id     :integer
 #  year           :integer
 #  path           :integer          is an Array
@@ -25,16 +24,24 @@ class DownloadFlow < Api::V3::Readonly::BaseModel
         self.table_name = 'download_flows'
 
         class << self
+          # @param options
+          # @option options [Boolean] :skip_dependencies skip refreshing
+          # @option options [Boolean] :skip_dependents skip refreshing
+          # @option options [Boolean] :skip_precompute skip precomputing downloads
           def refresh_now(options = {})
             refresh_dependencies(options) unless options[:skip_dependencies]
             Api::V3::TablePartitions.create
             after_refresh(options)
             refresh_dependents(options) unless options[:skip_dependents]
           end
 
+          # @param options
+          # @option options [Boolean] :skip_dependencies skip refreshing
+          # @option options [Boolean] :skip_dependents skip refreshing
+          # @option options [Boolean] :skip_precompute skip precomputing downloads
           # this materialized view takes a long time to refresh
-          def refresh_later(_options = {})
-            TablePartitionsWorker.perform_async
+          def refresh_later(options = {})
+            TablePartitionsWorker.perform_async(options)
           end
 
           protected

diff --git a/app/services/api/v3/download/flow_download_flat_query.rb b/app/services/api/v3/download/flow_download_flat_query.rb
@@ -17,7 +17,7 @@ def all
 
         # Calculating the total is too expensive, use an estimation instead
         def total
-          @size_query.sum(:count)
+          @size_query.sum('count::INT')
         end
 
         MAX_SIZE = 500_000

diff --git a/app/services/api/v3/download/flow_download_pivot_query.rb b/app/services/api/v3/download/flow_download_pivot_query.rb
@@ -27,6 +27,7 @@ def initialize_query
 
         def pivot_select_columns
           [
+            'rank() OVER (ORDER BY path, year)::int AS rn',
             'path',
             'year AS "YEAR"'
           ] + @path_columns +
@@ -57,6 +58,7 @@ def outer_select_columns
 
         def crosstab_columns
           [
+            'rn int',
             'path INT[]',
             '"YEAR" int'
           ] + @path_crosstab_columns + [

diff --git a/app/workers/table_partitions_worker.rb b/app/workers/table_partitions_worker.rb
@@ -7,7 +7,11 @@ class TablePartitionsWorker
                   run_lock_expiration: 150, # 2.5 mins
                   log_duplicate_payload: true
 
-  def perform
-    Api::V3::TablePartitions.create
+  # @param options
+  # @option options [Boolean] :skip_dependencies skip refreshing
+  # @option options [Boolean] :skip_dependents skip refreshing
+  # @option options [Boolean] :skip_precompute skip precomputing downloads
+  def perform(options)
+    Api::V3::Readonly::DownloadFlow.refresh_now(options.symbolize_keys)
   end
 end
diff --git a/db/migrate/20190801121907_aggregate_flows_for_download.rb b/db/migrate/20190801121907_aggregate_flows_for_download.rb
@@ -0,0 +1,13 @@
+class AggregateFlowsForDownload < ActiveRecord::Migration[5.2]
+  def change
+    update_view :download_flows_v,
+      version: 2,
+      revert_to_version: 1,
+      materialized: false
+
+    remove_column :download_flows, :id
+
+    # this takes a long time
+    Api::V3::Readonly::DownloadFlow.refresh_now
+  end
+end