Merge tag '6.0.0-rc'

Q3 data release + map upgrade & partial refreshing
Vizzuality · Oct 15, 2020 · 4778ea8 · 4778ea8
2 parents d9a91d8 + a8395c5
commit 4778ea8
Show file tree

Hide file tree

Showing 432 changed files with 20,078 additions and 9,789 deletions.
diff --git a/app/models/api/public/readonly/flow_attribute.rb b/app/models/api/public/readonly/flow_attribute.rb
@@ -1,25 +1,25 @@
 # == Schema Information
 #
-# Table name: flow_attributes_mv
+# Table name: flow_attributes
 #
-#  attribute_id :bigint(8)
-#  name         :text
-#  display_name :text
-#  unit         :text
-#  unit_type    :text
-#  context_id   :integer
-#  years        :integer          is an Array
+#  attribute_id    :integer          not null
+#  context_id      :integer          not null
+#  years           :integer          is an Array
+#  name            :text
+#  display_name    :text
+#  unit            :text
+#  unit_type       :text
+#  distinct_values :text             is an Array
 #
-# Indexes
-#
-#  flow_attributes_mv_attribute_id_context_id_idx  (attribute_id,context_id) UNIQUE
-#
-
 module Api
   module Public
     module Readonly
       class FlowAttribute < Api::Readonly::BaseModel
-        self.table_name = 'flow_attributes_mv'
+        include Api::V3::Readonly::MaterialisedTable
+
+        self.table_name = 'flow_attributes'
+
+        INDEXES = [].freeze
 
         belongs_to :context
 

diff --git a/app/models/api/readonly/base_model.rb b/app/models/api/readonly/base_model.rb
@@ -1,14 +1,14 @@
 module Api
   module Readonly
     class BaseModel < Api::V3::BaseModel
+
       def readonly?
         true
       end
 
       class << self
         # Refreshes the views dependencies, the view itself and its dependents
         # @param options
-        # @option options [Boolean] :skip_dependencies skip refreshing
         # @option options [Boolean] :skip_dependents skip refreshing
         # @option options [Boolean] :sync synchronously
         def refresh(options = {})
@@ -27,15 +27,14 @@ def refresh(options = {})
         end
 
         def refresh_now(options = {})
-          refresh_dependencies(options) unless options[:skip_dependencies]
           refresh_by_name(table_name, options)
           after_refresh(options)
           refresh_dependents(options) unless options[:skip_dependents]
         end
 
         # this materialized view takes a long time to refresh
         def refresh_later(options = {})
-          MaterializedViewRefreshWorker.perform_async(name, options)
+          FullRefreshWorker.perform_async(name, options)
         end
 
         protected
@@ -44,30 +43,9 @@ def long_running?
           false
         end
 
-        def unique_index?
-          true
-        end
-
-        # Refreshes materialized views this view depends on
-        def refresh_dependencies(options = {}); end
-
         # Refreshes materialized views that depend on this view
         def refresh_dependents(options = {}); end
 
-        def refresh_by_name(table_name, options)
-          safe_concurrently = true
-          # rubocop:disable Style/DoubleNegation
-          safe_concurrently = !!options[:concurrently] unless options[:concurrently].nil?
-          # rubocop:enable Style/DoubleNegation
-          is_populated = Api::V3::IsMviewPopulated.new(table_name).call
-          # cannot refresh concurrently a view that is not populated
-          safe_concurrently = false unless is_populated and unique_index?
-          Scenic.database.refresh_materialized_view(
-            table_name,
-            concurrently: safe_concurrently
-          )
-        end
-
         # Whatever needs doing after refreshing that is not a cascading refresh
         def after_refresh(_options = {}); end
       end

diff --git a/app/models/api/v3/blue_table.rb b/app/models/api/v3/blue_table.rb
@@ -1,5 +1,4 @@
 # See the README file for info on what's a blue table
-
 module Api
   module V3
     class BlueTable < BaseModel

diff --git a/app/models/api/v3/carto_layer.rb b/app/models/api/v3/carto_layer.rb
@@ -16,7 +16,6 @@
 #
 #  fk_rails_...  (contextual_layer_id => contextual_layers.id) ON DELETE => cascade
 #
-
 module Api
   module V3
     class CartoLayer < YellowTable

diff --git a/app/models/api/v3/chart.rb b/app/models/api/v3/chart.rb
@@ -20,7 +20,6 @@
 #  fk_rails_...  (parent_id => charts.id) ON DELETE => cascade
 #  fk_rails_...  (profile_id => profiles.id) ON DELETE => cascade
 #
-
 module Api
   module V3
     class Chart < YellowTable
@@ -163,15 +162,13 @@ def refresh_actor_basic_attributes
 
       def update_node_with_flows_actor_basic_attributes(profile_id)
         profile = Api::V3::Profile.find(profile_id)
-        context_node_type = profile.context_node_type
-        context = context_node_type.context
-        nodes = context_node_type.node_type.nodes
-        node_with_flows = Api::V3::Readonly::NodeWithFlows.
+        nodes_ids = Api::V3::Readonly::NodeWithFlows.
+          where(context_node_type_id: profile.context_node_type_id).
           without_unknowns.
           without_domestic.
-          where(context_id: context.id, id: nodes.map(&:id))
-        NodeWithFlowsRefreshActorBasicAttributesWorker.new.perform(
-          node_with_flows.map(&:id)
+          pluck(:id)
+        NodeWithFlowsRefreshActorBasicAttributesWorker.perform_async(
+          nodes_ids.uniq
         )
       end
 

diff --git a/app/models/api/v3/chart_attribute.rb b/app/models/api/v3/chart_attribute.rb
@@ -7,7 +7,7 @@
 #  position(Display order in scope of chart)                                                                                                     :integer
 #  years(Array of years for which to show this attribute in scope of chart; empty (NULL) for all years)                                          :integer          is an Array
 #  display_name(Name of attribute for display in chart)                                                                                          :text
-#  legend_name(Legend title)                                                                                                                     :text
+#  legend_name(Legend title; you can use {{commodity_name}}, {{company_name}}, {{jurisdiction_name}} and {{year}})                               :text
 #  display_type(Type of display, only used for trajectory deforestation plot in place profiles; e.g. area, line)                                 :text
 #  display_style(Style of display, only used for trajectory deforestation plot in place profiles; e.g. area-pink, area-black, line-dashed-black) :text
 #  state_average(Only used for trajectory deforestation plot in place profiles)                                                                  :boolean          default(FALSE), not null
@@ -23,7 +23,6 @@
 #
 #  fk_rails_...  (chart_id => charts.id) ON DELETE => cascade
 #
-
 module Api
   module V3
     class ChartAttribute < YellowTable
@@ -85,7 +84,6 @@ def self.yellow_foreign_keys
       end
 
       def refresh_dependents
-        Api::V3::Readonly::ChartAttribute.refresh
         refresh_actor_basic_attributes
       end
 
@@ -111,15 +109,15 @@ def refresh_actor_basic_attributes
       def update_node_with_flows_actor_basic_attributes(chart_id)
         chart = Api::V3::Chart.find(chart_id)
         profile = chart.profile
-        context_node_type = profile.context_node_type
-        context = context_node_type.context
-        nodes = context_node_type.node_type.nodes
-        node_with_flows = Api::V3::Readonly::NodeWithFlows.
+        nodes_ids = Api::V3::Readonly::NodeWithFlows.
+          where(
+            context_node_type_id: profile.context_node_type_id
+          ).
           without_unknowns.
           without_domestic.
-          where(context_id: context.id, id: nodes.map(&:id))
-        NodeWithFlowsRefreshActorBasicAttributesWorker.new.perform(
-          node_with_flows.map(&:id)
+          pluck(:id)
+        NodeWithFlowsRefreshActorBasicAttributesWorker.perform_async(
+          nodes_ids.uniq
         )
       end
 

diff --git a/app/models/api/v3/chart_ind.rb b/app/models/api/v3/chart_ind.rb
@@ -15,7 +15,6 @@
 #  fk_rails_...  (chart_attribute_id => chart_attributes.id) ON DELETE => cascade
 #  fk_rails_...  (ind_id => inds.id) ON DELETE => cascade ON UPDATE => cascade
 #
-
 module Api
   module V3
     class ChartInd < YellowTable

diff --git a/app/models/api/v3/chart_qual.rb b/app/models/api/v3/chart_qual.rb
@@ -17,7 +17,6 @@
 #  fk_rails_...  (chart_attribute_id => chart_attributes.id) ON DELETE => cascade
 #  fk_rails_...  (qual_id => quals.id) ON DELETE => cascade ON UPDATE => cascade
 #
-
 module Api
   module V3
     class ChartQual < YellowTable

diff --git a/app/models/api/v3/chart_quant.rb b/app/models/api/v3/chart_quant.rb
@@ -17,7 +17,6 @@
 #  fk_rails_...  (chart_attribute_id => chart_attributes.id) ON DELETE => cascade
 #  fk_rails_...  (quant_id => quants.id) ON DELETE => cascade ON UPDATE => cascade
 #
-
 module Api
   module V3
     class ChartQuant < YellowTable

diff --git a/app/models/api/v3/commodity.rb b/app/models/api/v3/commodity.rb
@@ -9,7 +9,6 @@
 #
 #  commodities_name_key  (name) UNIQUE
 #
-
 module Api
   module V3
     class Commodity < BlueTable

diff --git a/app/models/api/v3/context.rb b/app/models/api/v3/context.rb
@@ -19,7 +19,6 @@
 #  fk_rails_...  (commodity_id => commodities.id) ON DELETE => cascade ON UPDATE => cascade
 #  fk_rails_...  (country_id => countries.id) ON DELETE => cascade ON UPDATE => cascade
 #
-
 module Api
   module V3
     class Context < BlueTable

diff --git a/app/models/api/v3/context_node_type.rb b/app/models/api/v3/context_node_type.rb
@@ -18,7 +18,6 @@
 #  fk_rails_...  (context_id => contexts.id) ON DELETE => cascade
 #  fk_rails_...  (node_type_id => node_types.id) ON DELETE => cascade
 #
-
 module Api
   module V3
     class ContextNodeType < BlueTable

diff --git a/app/models/api/v3/context_node_type_property.rb b/app/models/api/v3/context_node_type_property.rb
@@ -11,19 +11,17 @@
 #  role(A grouping which defines in which filtering panel to display nodes)                                                                     :string           not null
 #  prefix(Used to construct the summary sentence of selection criteria)                                                                         :text             not null
 #  geometry_context_node_type_id(Use for geo columns, when geometry is to be taken from another node type (e.g. logistics hub -> municipality)) :integer
-#  is_visible                                                                                                                                   :boolean          default(TRUE), not null
+#  is_visible(When set, show this node type in sankey & dashboards)                                                                             :boolean          default(TRUE), not null
 #
 # Indexes
 #
-#  context_node_type_properties_context_node_type_id_idx  (context_node_type_id)
 #  context_node_type_properties_context_node_type_id_key  (context_node_type_id) UNIQUE
 #
 # Foreign Keys
 #
 #  fk_rails_...  (context_node_type_id => context_node_types.id) ON DELETE => cascade
 #  fk_rails_...  (geometry_context_node_type_id => context_node_types.id) ON DELETE => nullify
 #
-
 module Api
   module V3
     class ContextNodeTypeProperty < YellowTable
@@ -60,49 +58,13 @@ class ContextNodeTypeProperty < YellowTable
       validate :geometry_context_node_type_from_same_context
       validate :geometry_context_node_type_is_geo_column
 
-      after_commit :refresh_dependents
-
       def self.blue_foreign_keys
         [
           {name: :context_node_type_id, table_class: Api::V3::ContextNodeType},
           {name: :geometry_context_node_type_id, table_class: Api::V3::ContextNodeType, nullable: true}
         ]
       end
 
-      def refresh_dependents
-        Api::V3::Readonly::Context.refresh
-        if previous_changes.key?('is_geo_column')
-          Api::V3::Readonly::NodeWithFlowsOrGeo.refresh_later
-        end
-        return unless previous_changes.key?('role')
-
-        refresh_role_dependents(previous_changes['role'])
-      end
-
-      def refresh_role_dependents(roles = [])
-        return unless roles.any?
-
-        Api::V3::Readonly::NodeWithFlows.refresh_later
-
-        if roles.include?(SOURCE_ROLE)
-          Api::V3::Readonly::Dashboards::Source.refresh_later
-        end
-        # TODO: remove once dashboards_companies_mv retired
-        if (roles & [EXPORTER_ROLE, IMPORTER_ROLE]).any?
-          Api::V3::Readonly::Dashboards::Company.refresh_later
-        end
-        # END TODO
-        if roles.include?(EXPORTER_ROLE)
-          Api::V3::Readonly::Dashboards::Exporter.refresh_later
-        end
-        if roles.include?(IMPORTER_ROLE)
-          Api::V3::Readonly::Dashboards::Importer.refresh_later
-        end
-        if roles.include?(DESTINATION_ROLE)
-          Api::V3::Readonly::Dashboards::Destination.refresh_later
-        end
-      end
-
       def self.roles
         ROLES
       end

diff --git a/app/models/api/v3/context_property.rb b/app/models/api/v3/context_property.rb
@@ -17,7 +17,6 @@
 #
 #  fk_rails_...  (context_id => contexts.id) ON DELETE => cascade
 #
-
 module Api
   module V3
     class ContextProperty < YellowTable
@@ -36,17 +35,11 @@ class ContextProperty < YellowTable
       validates :is_highlighted, inclusion: {in: [true, false]}
       validates :default_basemap, inclusion: {in: DEFAULT_BASEMAP, allow_blank: true}
 
-      after_commit :refresh_dependents
-
       def self.blue_foreign_keys
         [
           {name: :context_id, table_class: Api::V3::Context}
         ]
       end
-
-      def refresh_dependents
-        Api::V3::Readonly::Context.refresh
-      end
     end
   end
 end
diff --git a/app/models/api/v3/contextual_layer.rb b/app/models/api/v3/contextual_layer.rb
@@ -21,7 +21,6 @@
 #
 #  fk_rails_...  (context_id => contexts.id) ON DELETE => cascade
 #
-
 module Api
   module V3
     class ContextualLayer < YellowTable

diff --git a/app/models/api/v3/countries_com_trade_partner_indicator.rb b/app/models/api/v3/countries_com_trade_partner_indicator.rb
@@ -16,7 +16,7 @@
 #
 module Api
   module V3
-    class CountriesComTradePartnerIndicator < YellowTable
+    class CountriesComTradePartnerIndicator < Api::V3::BaseModel
       belongs_to :commodity, optional: false
 
       validates :iso3, presence: true

diff --git a/app/models/api/v3/countries_com_trade_world_indicator.rb b/app/models/api/v3/countries_com_trade_world_indicator.rb
@@ -14,7 +14,7 @@
 #
 module Api
   module V3
-    class CountriesComTradeWorldIndicator < YellowTable
+    class CountriesComTradeWorldIndicator < Api::V3::BaseModel
       belongs_to :commodity, optional: false
 
       validates :iso3, presence: true

diff --git a/app/models/api/v3/country.rb b/app/models/api/v3/country.rb
@@ -10,7 +10,6 @@
 #
 #  countries_iso2_key  (iso2) UNIQUE
 #
-
 module Api
   module V3
     class Country < BlueTable

diff --git a/app/models/api/v3/country_property.rb b/app/models/api/v3/country_property.rb
@@ -18,7 +18,6 @@
 #
 #  fk_rails_...  (country_id => countries.id) ON DELETE => cascade ON UPDATE => cascade
 #
-
 module Api
   module V3
     class CountryProperty < YellowTable

diff --git a/app/models/api/v3/dashboard_template.rb b/app/models/api/v3/dashboard_template.rb
@@ -11,7 +11,6 @@
 #  image_updated_at   :datetime
 #  category           :string
 #
-
 module Api
   module V3
     class DashboardTemplate < YellowTable

diff --git a/app/models/api/v3/dashboard_template_commodity.rb b/app/models/api/v3/dashboard_template_commodity.rb
@@ -6,7 +6,6 @@
 #  dashboard_template_id :integer
 #  commodity_id          :integer
 #
-
 module Api
   module V3
     class DashboardTemplateCommodity < YellowTable