Merge pull request #212 from pact-foundation/feat/optimise-index-with…

…-tags

feat: optimise index with tags

db/migrations/20180311_optimise_head_matrix.rb

@@ -16,6 +16,7 @@
     )
 
     # Add provider_version_order to original definition
+    # The most recent verification for each pact version
     v = :verifications
     create_or_replace_view(:latest_verifications,
       from(v)

db/migrations/20180523_create_latest_verifications_for_consumer_version_tags.rb

@@ -0,0 +1,50 @@
+Sequel.migration do
+  up do
+    # The latest verification id for each consumer version tag
+    create_view(:latest_verification_ids_for_consumer_version_tags,
+      "select
+        pv.pacticipant_id as provider_id,
+        lpp.consumer_id,
+        t.name as consumer_version_tag_name,
+        max(v.id) as latest_verification_id
+      from verifications v
+      join latest_pact_publications_by_consumer_versions lpp
+        on v.pact_version_id = lpp.pact_version_id
+      join tags t
+        on lpp.consumer_version_id = t.version_id
+      join versions pv
+        on v.provider_version_id = pv.id
+      group by pv.pacticipant_id, lpp.consumer_id, t.name")
+
+    # The most recent verification for each consumer/consumer version tag/provider
+    latest_verifications = from(:verifications)
+      .select(
+        Sequel[:lv][:consumer_id],
+        Sequel[:lv][:provider_id],
+        Sequel[:lv][:consumer_version_tag_name],
+        Sequel[:pv][:sha].as(:pact_version_sha),
+        Sequel[:prv][:number].as(:provider_version_number),
+        Sequel[:prv][:order].as(:provider_version_order),
+        )
+      .select_append{ verifications.* }
+      .join(:latest_verification_ids_for_consumer_version_tags,
+        {
+          Sequel[:verifications][:id] => Sequel[:lv][:latest_verification_id],
+        }, { table_alias: :lv })
+      .join(:versions,
+        {
+          Sequel[:verifications][:provider_version_id] => Sequel[:prv][:id]
+        }, { table_alias: :prv })
+      .join(:pact_versions,
+        {
+          Sequel[:verifications][:pact_version_id] => Sequel[:pv][:id]
+        }, { table_alias: :pv })
+
+    create_or_replace_view(:latest_verifications_for_consumer_version_tags, latest_verifications)
+  end
+
+  down do
+    drop_view(:latest_verifications_for_consumer_version_tags)
+    drop_view(:latest_verification_ids_for_consumer_version_tags)
+  end
+end

db/migrations/20180524_create_latest_verifications_for_consumer_and_provider.rb

@@ -0,0 +1,46 @@
+Sequel.migration do
+  up do
+    # The latest verification id for each consumer version tag
+    create_view(:latest_verification_ids_for_consumer_and_provider,
+      "select
+        pv.pacticipant_id as provider_id,
+        lpp.consumer_id,
+        max(v.id) as latest_verification_id
+      from verifications v
+      join latest_pact_publications_by_consumer_versions lpp
+        on v.pact_version_id = lpp.pact_version_id
+      join versions pv
+        on v.provider_version_id = pv.id
+      group by pv.pacticipant_id, lpp.consumer_id")
+
+    # The most recent verification for each consumer/consumer version tag/provider
+    latest_verifications = from(:verifications)
+      .select(
+        Sequel[:lv][:consumer_id],
+        Sequel[:lv][:provider_id],
+        Sequel[:pv][:sha].as(:pact_version_sha),
+        Sequel[:prv][:number].as(:provider_version_number),
+        Sequel[:prv][:order].as(:provider_version_order),
+        )
+      .select_append{ verifications.* }
+      .join(:latest_verification_ids_for_consumer_and_provider,
+        {
+          Sequel[:verifications][:id] => Sequel[:lv][:latest_verification_id],
+        }, { table_alias: :lv })
+      .join(:versions,
+        {
+          Sequel[:verifications][:provider_version_id] => Sequel[:prv][:id]
+        }, { table_alias: :prv })
+      .join(:pact_versions,
+        {
+          Sequel[:verifications][:pact_version_id] => Sequel[:pv][:id]
+        }, { table_alias: :pv })
+
+    create_or_replace_view(:latest_verifications_for_consumer_and_provider, latest_verifications)
+  end
+
+  down do
+    drop_view(:latest_verifications_for_consumer_and_provider)
+    drop_view(:latest_verification_ids_for_consumer_and_provider)
+  end
+end

lib/pact_broker/index/service.rb

@@ -2,6 +2,7 @@
 require 'pact_broker/logging'
 require 'pact_broker/domain/index_item'
 require 'pact_broker/matrix/head_row'
+require 'pact_broker/matrix/aggregated_row'
 
 module PactBroker
 
@@ -13,107 +14,37 @@ class Service
       extend PactBroker::Logging
 
       def self.find_index_items options = {}
-        rows = []
-        overall_latest_publication_ids = nil
-
         rows = PactBroker::Matrix::HeadRow
           .select_all_qualified
           .eager(:latest_triggered_webhooks)
           .eager(:webhooks)
-          .order(:consumer_name, :provider_name)
-          .eager(:consumer_version_tags)
-          .eager(:provider_version_tags)
 
         if !options[:tags]
-          rows = rows.where(consumer_version_tag_name: nil).all
-          overall_latest_publication_ids = rows.collect(&:pact_publication_id)
-        end
-
-        if options[:tags]
+          rows = rows.where(consumer_version_tag_name: nil)
+        else
           if options[:tags].is_a?(Array)
             rows = rows.where(consumer_version_tag_name: options[:tags]).or(consumer_version_tag_name: nil)
           end
-
-          rows = rows.all
-          overall_latest_publication_ids = rows.select{|r| !r[:consumer_version_tag_name] }.collect(&:pact_publication_id).uniq
-
-          # Smoosh all the rows with matching pact publications together
-          # and collect their consumer_head_tag_names
-          rows = rows
-            .group_by(&:pact_publication_id)
-            .values
-            .collect{|group| [group.last, group.collect{|r| r[:consumer_version_tag_name]}.compact] }
-            .collect{ |(row, tag_names)| row.consumer_head_tag_names = tag_names; row }
+          rows = rows.eager(:consumer_version_tags)
+                .eager(:provider_version_tags)
+                .eager(:latest_verification_for_consumer_version_tag)
         end
+        rows = rows.all.group_by(&:pact_publication_id).values.collect{ | rows| Matrix::AggregatedRow.new(rows) }
 
-        index_items = []
-        rows.sort.each do | row |
-          tag_names = []
-          if options[:tags]
-            tag_names = row.consumer_version_tags.collect(&:name)
-          end
-
-          overall_latest = overall_latest_publication_ids.include?(row.pact_publication_id)
-          latest_verification = if overall_latest
-            verification_repository.find_latest_verification_for row.consumer_name, row.provider_name
-          else
-            tag_names.collect do | tag_name |
-              verification_repository.find_latest_verification_for row.consumer_name, row.provider_name, tag_name
-            end.compact.sort do | v1, v2 |
-              # Some provider versions have nil orders, not sure why
-              # Sort by execution_date instead of order
-              v1.execution_date <=> v2.execution_date
-            end.last
-          end
-
-          index_items << PactBroker::Domain::IndexItem.create(
+        rows.sort.collect do | row |
+          # TODO simplify. Do we really need 3 layers of abstraction?
+          PactBroker::Domain::IndexItem.create(
             row.consumer,
             row.provider,
             row.pact,
-            overall_latest,
-            latest_verification,
+            row.overall_latest?,
+            row.latest_verification,
             row.webhooks,
             row.latest_triggered_webhooks,
-            row.consumer_head_tag_names,
-            row.provider_version_tags.select(&:latest?)
+            options[:tags] ? row.consumer_head_tag_names : [],
+            options[:tags] ? row.provider_version_tags.select(&:latest?) : []
           )
         end
-
-        index_items
-      end
-
-      def self.tags_for(pact, options)
-        if options[:tags] == true
-          tag_service.find_all_tag_names_for_pacticipant(pact.consumer_name)
-        elsif options[:tags].is_a?(Array)
-          options[:tags]
-        else
-          []
-        end
-      end
-
-      def self.build_index_item_rows(pact, tags)
-        index_items = [build_latest_pact_index_item(pact, tags)]
-        tags.each do | tag |
-          index_items << build_index_item_for_tagged_pact(pact, tag)
-        end
-        index_items.compact
-      end
-
-      def self.build_latest_pact_index_item pact, tags
-        latest_verification = verification_service.find_latest_verification_for(pact.consumer, pact.provider)
-        webhooks = webhook_service.find_by_consumer_and_provider pact.consumer, pact.provider
-        triggered_webhooks = webhook_service.find_latest_triggered_webhooks pact.consumer, pact.provider
-        tag_names = pact.consumer_version_tag_names.select{ |name| tags.include?(name) }
-        PactBroker::Domain::IndexItem.create pact.consumer, pact.provider, pact, true, latest_verification, webhooks, triggered_webhooks, tag_names
-      end
-
-      def self.build_index_item_for_tagged_pact latest_pact, tag
-        pact = pact_service.find_latest_pact consumer_name: latest_pact.consumer_name, provider_name: latest_pact.provider_name, tag: tag
-        return nil unless pact
-        return nil if pact.id == latest_pact.id
-        verification = verification_repository.find_latest_verification_for pact.consumer_name, pact.provider_name, tag
-        PactBroker::Domain::IndexItem.create pact.consumer, pact.provider, pact, false, verification, [], [], [tag]
       end
     end
   end

lib/pact_broker/matrix/aggregated_row.rb

@@ -0,0 +1,71 @@
+require 'pact_broker/verifications/repository'
+
+# A collection of matrix rows with the same pact publication id
+# It's basically a normalised view of a denormalised view :(
+# A pact publication may be the overall latest, and/or the latest for a tag
+module PactBroker
+  module Matrix
+    class AggregatedRow
+      extend Forwardable
+
+      delegate [:consumer, :consumer_name, :consumer_version, :consumer_version_number, :consumer_version_order, :consumer_version_tags] => :first_row
+      delegate [:provider, :provider_name, :provider_version, :provider_version_number, :provider_version_order, :provider_version_tags] => :first_row
+      delegate [:pact, :pact_version, :pact_revision_number, :webhooks, :latest_triggered_webhooks, :'<=>'] => :first_row
+      delegate [:verification_id, :verification] => :first_row
+
+      def initialize matrix_rows
+        @matrix_rows = matrix_rows
+        @first_row = matrix_rows.first
+      end
+
+      def overall_latest?
+        !!matrix_rows.find{ | row| row.consumer_version_tag_name.nil? }
+      end
+
+      # If this comes back nil, it won't be "cached", but it's a reasonably
+      # quick query, so it's probably ok.
+      def latest_verification
+        @latest_verification ||= begin
+          verification = matrix_rows.collect do | row|
+              row.verification || latest_verification_for_consumer_version_tag(row)
+            end.compact.sort{ |v1, v2| v1.id <=> v2.id }.last
+
+          if !verification && overall_latest?
+            overall_latest_verification
+          else
+            verification
+          end
+        end
+      end
+
+      # The list of tag names for which this pact publication is the most recent with that tag
+      # There could, however, be a later consumer version that does't have a pact (perhaps because it was deleted)
+      # that has the same tag.
+      # TODO show a warning when the data is "corrupted" as above.
+      def consumer_head_tag_names
+        matrix_rows.collect(&:consumer_version_tag_name).compact
+      end
+
+      private
+
+      attr_reader :matrix_rows
+
+      def latest_verification_for_consumer_version_tag row
+        row.latest_verification_for_consumer_version_tag if row.consumer_version_tag_name
+      end
+
+      def overall_latest_verification
+        # not eager loaded because it shouldn't be called that often
+        first_row.latest_verification_for_consumer_and_provider
+      end
+
+      def first_row
+        @first_row
+      end
+
+      def consumer_version_tag_names
+        matrix_rows.collect(&:consumer_version_tag_name)
+      end
+    end
+  end
+end

lib/pact_broker/matrix/head_row.rb

@@ -7,6 +7,29 @@ module Matrix
     class HeadRow < Row
       set_dataset(:materialized_head_matrix)
 
+      # one_to_one :latest_verification_for_consumer_version_tag,
+      #   :class => "PactBroker::Verifications::LatestVerificationForConsumerVersionTag",
+      #   primary_key: [:provider_id, :consumer_id, :consumer_version_tag_name], key: [:provider_id, :consumer_id, :consumer_version_tag_name]
+
+      # Loading the latest_verification_for_consumer_version_tag objects this way is quicker than
+      # doing it using an inbult relation with primary_key/key, if we are loading the relation for
+      # the entire HeadRow table
+      # Using the inbuilt relation puts constraints on the columns that are not necessary and slow
+      # the query down.
+      # This relation relies on consumer_version_tags already being loaded
+      one_to_one :latest_verification_for_consumer_version_tag, :class => "PactBroker::Verifications::LatestVerificationForConsumerVersionTag", primary_keys: [], key: [], :eager_loader=>(proc do |eo_opts|
+        tag_to_row = eo_opts[:rows].each_with_object({}) { | row, map | map[[row.provider_id, row.consumer_id, row.consumer_version_tag_name]] = row }
+        eo_opts[:rows].each{|row| row.associations[:latest_verification_for_consumer_version_tag] = nil}
+
+        # Need the all then the each to ensure the eager loading works
+        PactBroker::Verifications::LatestVerificationForConsumerVersionTag.each do | verification |
+          key = [verification.provider_id, verification.consumer_id, verification.consumer_version_tag_name]
+          if tag_to_row[key]
+            tag_to_row[key].associations[:latest_verification_for_consumer_version_tag] = verification
+          end
+        end
+      end)
+
       dataset_module do
         include PactBroker::Repositories::Helpers
         include PactBroker::Logging