feature/gra-776-filter-postgres-constraints-on-available-schemas (#598)

* Postgres will only query for table constraints associated with table schemas it was able to read.

grai-integrations/source-postgres/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "grai_source_postgres"
-version = "0.2.0"
+version = "0.2.1"
 description = ""
 authors = ["Ian Eaves <ian@grai.io>"]
 license = "Elastic-2.0"

grai-integrations/source-postgres/src/grai_source_postgres/__init__.py

@@ -1,4 +1,4 @@
 from grai_source_postgres import adapters, base, loader, models, package_definitions
 from grai_source_postgres.package_definitions import config
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"

grai-integrations/source-postgres/src/grai_source_postgres/loader.py

@@ -249,6 +249,7 @@ def foreign_keys(self) -> List[Edge]:
         """
         # query is from https://dba.stackexchange.com/questions/36979/retrieving-all-pk-and-fk/37068#37068
         # Only need constraint_types == 'f' for foreign keys but the others might be useful someday.
+
         query = """
             SELECT c.conname                                         AS constraint_name,
                    c.contype                                         AS constraint_type,
@@ -268,13 +269,17 @@ def foreign_keys(self) -> List[Edge]:
                    LEFT JOIN pg_class f_tbl ON f_tbl.oid = c.confrelid
                    LEFT JOIN pg_namespace f_sch ON f_sch.oid = f_tbl.relnamespace
                    LEFT JOIN pg_attribute f_col ON (f_col.attrelid = f_tbl.oid AND f_col.attnum = f_u.attnum)
+            WHERE sch.nspname IN %(schemas)s AND f_sch.nspname IN %(schemas)s
             GROUP BY constraint_name, constraint_type, "self_schema", "self_table", definition, "foreign_schema", "foreign_table"
             ORDER BY "self_schema", "self_table";
         """
+
+        query_params = {"schemas": tuple({table.table_schema for table in self.tables})}
+        results = self.query_runner(query, query_params)
+
         addtl_args = {
             "namespace": self.namespace,
         }
-        results = self.query_runner(query)
         filtered_results = (result for result in results if result["constraint_type"] == "f")
         result = (EdgeQuery(**fk, **addtl_args).to_edge() for fk in filtered_results)
         return [r for r in result if r is not None]

grai-integrations/source-postgres/tests/conftest.py

@@ -52,17 +52,16 @@ def integration(conn_credentials) -> PostgresIntegration:
 
 
 @pytest.fixture
-def nodes_and_edges(connection):
-    return [], []
-    # """
+def nodes_and_edges(integration):
+    nodes, edges = integration.get_nodes_and_edges()
+    return nodes, edges
 
-    # Args:
-    #     connection:
 
-    # Returns:
+@pytest.fixture
+def nodes(nodes_and_edges):
+    return nodes_and_edges[0]
 
-    # Raises:
 
-    # """
-    # nodes, edges = get_nodes_and_edges(connection, "v1")
-    # return nodes, edges
+@pytest.fixture
+def edges(nodes_and_edges):
+    return nodes_and_edges[1]

grai-integrations/source-postgres/tests/test_adapters.py

@@ -151,6 +151,20 @@ def test_edge_adapter(item, version, target, mock_source):
     assert isinstance(result, target)
 
 
+def test_all_sources_in_nodes(nodes, edges):
+    edge_ids = {(e.spec.source.name, e.spec.source.namespace) for e in edges}
+    node_ids = {(n.spec.name, n.spec.namespace) for n in nodes}
+    missing_sources = edge_ids - node_ids
+    assert not missing_sources
+
+
+def test_all_destinations_in_nodes(nodes, edges):
+    edge_ids = {(e.spec.destination.name, e.spec.destination.namespace) for e in edges}
+    node_ids = {(n.spec.name, n.spec.namespace) for n in nodes}
+    missing_destinations = edge_ids - node_ids
+    assert not missing_destinations
+
+
 # def test_metadata_has_core_metadata_ids(nodes_and_edges):
 #     """
 #

grai-server/app/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "the_guide"
-version = "0.1.37"
+version = "0.1.38"
 description = ""
 authors = ["Ian Eaves <ian@grai.io>", "Edward Louth <edward@grai.io>"]
 license = "Elastic-2.0"
@@ -48,7 +48,7 @@ grai-source-looker = "^0.0.2"
 grai-source-metabase = "^0.2.2"
 grai-source-mssql = "^0.1.0"
 grai-source-mysql =  "^0.1.0"
-grai-source-postgres = "^0.2.0"
+grai-source-postgres = "^0.2.1"
 grai-source-redshift =  "^0.1.0"
 grai-source-snowflake = "^0.1.0"
 algoliasearch-django = "^2.0.0"

grai-server/app/the_guide/__init__.py

@@ -2,5 +2,5 @@
 # Django starts so that shared_task will use this app.
 from .celery import app as celery_app
 
-__version__ = "0.1.37"
+__version__ = "0.1.38"
 __all__ = ("celery_app",)