refactor(matrix): improve performance of matrix queries

pact-foundation · Feb 6, 2018 · ecfca29 · ecfca29
1 parent 6ae039d
commit ecfca29
Show file tree

Hide file tree

Showing 4 changed files with 121 additions and 3 deletions.
diff --git a/db/migrations/20180206_recreate_head_matrix_rename_consumer_tag_name.rb b/db/migrations/20180206_recreate_head_matrix_rename_consumer_tag_name.rb
@@ -30,9 +30,6 @@
     alter_table(:materialized_head_matrix) do
       rename_column(:consumer_tag_name, :consumer_version_tag_name)
     end
-
-    from(:materialized_head_matrix).delete
-    from(:materialized_head_matrix).insert(from(:head_matrix).select_all)
   end
 
   down do

diff --git a/db/migrations/20180207_recreate_head_matrix_union_all.rb b/db/migrations/20180207_recreate_head_matrix_union_all.rb
@@ -0,0 +1,63 @@
+Sequel.migration do
+  up do
+    # a row for each of the latest pact publications,
+    # and a row for each of the latest tagged pact publications
+    drop_view(:head_matrix)
+    create_view(:head_matrix,
+      "SELECT matrix.*, hpp.tag_name as consumer_version_tag_name
+      FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN head_pact_publications hpp
+      ON matrix.consumer_id = hpp.consumer_id
+      AND matrix.provider_id = hpp.provider_id
+      AND matrix.consumer_version_order = hpp.consumer_version_order
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider AS lv
+      ON matrix.consumer_version_id = lv.consumer_version_id
+      AND matrix.provider_id = lv.provider_id
+      AND matrix.verification_id = lv.latest_verification_id
+
+      UNION ALL
+
+      SELECT matrix.*, hpp.tag_name as consumer_version_tag_name
+      FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN head_pact_publications hpp
+      ON matrix.consumer_id = hpp.consumer_id
+      AND matrix.provider_id = hpp.provider_id
+      AND matrix.consumer_version_order = hpp.consumer_version_order
+      where verification_id is null
+      "
+    )
+
+    from(:materialized_head_matrix).delete
+    from(:materialized_head_matrix).insert(from(:head_matrix).select_all)
+  end
+
+  down do
+    drop_view(:head_matrix)
+    create_view(:head_matrix,
+      "SELECT matrix.*, hpp.tag_name as consumer_version_tag_name
+      FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN head_pact_publications hpp
+      ON matrix.consumer_id = hpp.consumer_id
+      AND matrix.provider_id = hpp.provider_id
+      AND matrix.consumer_version_order = hpp.consumer_version_order
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider AS lv
+      ON matrix.consumer_version_id = lv.consumer_version_id
+      AND matrix.provider_id = lv.provider_id
+      AND matrix.verification_id = lv.latest_verification_id
+
+      UNION
+
+      SELECT matrix.*, hpp.tag_name as consumer_version_tag_name
+      FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN head_pact_publications hpp
+      ON matrix.consumer_id = hpp.consumer_id
+      AND matrix.provider_id = hpp.provider_id
+      AND matrix.consumer_version_order = hpp.consumer_version_order
+      where verification_id is null
+      "
+    )
+
+    from(:materialized_head_matrix).delete
+    from(:materialized_head_matrix).insert(from(:head_matrix).select_all)
+  end
+end
diff --git a/db/migrations/20180208_add_cv_tag_name_index_to_mat_head_matrix.rb b/db/migrations/20180208_add_cv_tag_name_index_to_mat_head_matrix.rb
@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:materialized_head_matrix) do
+      add_index(:consumer_version_tag_name)
+    end
+  end
+end
diff --git a/db/migrations/20180209_recreate_latest_matrix_for_cv_and_pv_union_all.rb b/db/migrations/20180209_recreate_latest_matrix_for_cv_and_pv_union_all.rb
@@ -0,0 +1,51 @@
+Sequel.migration do
+  up do
+    # Removes 'overwritten' pacts and verifications from the matrix
+    # (ie. only show latest pact revision for each consumer version and
+    # latest verification for each provider version)
+    # Must include lines where verification_id is null so that we don't
+    # lose the unverified pacts.
+    # In this view there will be one row for each consumer version/provider version
+    # This view used to be (stupidly) called latest_matrix
+    create_or_replace_view(:latest_matrix_for_consumer_version_and_provider_version,
+      "SELECT matrix.* FROM matrix
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv
+      ON ((matrix.consumer_version_id = lv.consumer_version_id)
+      AND (matrix.provider_version_id = lv.provider_version_id)
+      AND ((matrix.verification_id = lv.latest_verification_id)))
+
+      UNION ALL
+
+      select matrix.* from matrix
+      inner join latest_pact_publication_revision_numbers lr
+      on matrix.consumer_id = lr.consumer_id
+      and matrix.provider_id = lr.provider_id
+      and matrix.consumer_version_order = lr.consumer_version_order
+      and matrix.pact_revision_number = lr.latest_revision_number
+      where verification_id is null
+      "
+    )
+  end
+
+  down do
+    # revert to previous crappy definition
+    create_or_replace_view(:latest_matrix_for_consumer_version_and_provider_version,
+      "SELECT matrix.* FROM matrix
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv
+      ON ((matrix.consumer_version_id = lv.consumer_version_id)
+      AND (matrix.provider_version_id = lv.provider_version_id)
+      AND ((matrix.verification_id = lv.latest_verification_id)))
+
+      UNION
+
+      select matrix.* from matrix
+      inner join latest_pact_publication_revision_numbers lr
+      on matrix.consumer_id = lr.consumer_id
+      and matrix.provider_id = lr.provider_id
+      and matrix.consumer_version_order = lr.consumer_version_order
+      and matrix.pact_revision_number = lr.latest_revision_number
+      where verification_id is null
+      "
+    )
+  end
+end