Merge branch 'master' into feature/gra-705-mocked-column-statistics

grai-integrations/source-bigquery/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "grai_source_bigquery"
-version = "0.2.0"
+version = "0.2.1"
 description = ""
 authors = ["Edward Louth <edward@grai.io>"]
 license = "Elastic-2.0"

grai-integrations/source-bigquery/src/grai_source_bigquery/__init__.py

@@ -1,4 +1,4 @@
 from grai_source_bigquery import adapters, base, loader, models, package_definitions
 from grai_source_bigquery.package_definitions import config
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"

grai-integrations/source-bigquery/src/grai_source_bigquery/loader.py

@@ -61,8 +61,8 @@ def __init__(
     ):
         self.namespace = get_from_env("namespace", "default") if namespace is None else namespace
         self.project = get_from_env("project", required=False) if project is None else project
-        self.dataset = get_from_env("dataset", required=False) if dataset is None else dataset
-        self.datasets = [self.dataset] if isinstance(self.dataset, str) else self.dataset
+        dataset = get_from_env("dataset", required=False) if dataset is None else dataset
+        self.datasets = [dataset] if isinstance(dataset, str) else dataset
 
         self.credentials = get_from_env("credentials", required=False) if credentials is None else credentials
         self._connection: Optional[bigquery.connector.BigqueryConnection] = None
@@ -394,9 +394,7 @@ def logs(self) -> List[Any]:
             f' AND timestamp>="{yesterday.strftime(time_format)}"'
         )
 
-        datasets = [self.dataset] if isinstance(self.dataset, str) else self.dataset
-
-        for dataset in datasets:
+        for dataset in self.datasets:
             filter_str += (
                 f' AND NOT protoPayload.metadata.jobChange.job.jobStats.queryStats.referencedTables="projects/grai-demo/datasets/{dataset}/tables/INFORMATION_SCHEMA.TABLES"'
                 f' AND NOT protoPayload.metadata.jobInsertion.job.jobStats.queryStats.referencedTables="projects/grai-demo/datasets/{dataset}/tables/INFORMATION_SCHEMA.TABLES"'

grai-server/app/lineage/admin.py

@@ -8,6 +8,20 @@
 from connections.models import Connection
 
 from .models import Edge, Event, Filter, Node, Source
+from connections.models import Run
+
+
+@admin.action(description="Force delete selected sources")
+def delete_sources(modeladmin, request, queryset):  # pragma: no cover
+    sources = queryset
+
+    for source in sources:
+        for connection in source.connections.all():
+            Run.objects.filter(connection=connection).delete()
+
+            connection.delete()
+
+    queryset.delete()
 
 
 class EdgeInline(admin.TabularInline):
@@ -180,6 +194,10 @@ class SourceAdmin(admin.ModelAdmin):
         ConnectionInline,
     ]
 
+    actions = [
+        delete_sources,
+    ]
+
 
 admin.site.register(Node, NodeAdmin)
 admin.site.register(Edge, EdgeAdmin)