diff --git a/grai-integrations/source-bigquery/pyproject.toml b/grai-integrations/source-bigquery/pyproject.toml index 49fde9955..1e1b72741 100644 --- a/grai-integrations/source-bigquery/pyproject.toml +++ b/grai-integrations/source-bigquery/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "grai_source_bigquery" -version = "0.2.0" +version = "0.2.1" description = "" authors = ["Edward Louth "] license = "Elastic-2.0" diff --git a/grai-integrations/source-bigquery/src/grai_source_bigquery/__init__.py b/grai-integrations/source-bigquery/src/grai_source_bigquery/__init__.py index bf030e3d3..6230c6be9 100644 --- a/grai-integrations/source-bigquery/src/grai_source_bigquery/__init__.py +++ b/grai-integrations/source-bigquery/src/grai_source_bigquery/__init__.py @@ -1,4 +1,4 @@ from grai_source_bigquery import adapters, base, loader, models, package_definitions from grai_source_bigquery.package_definitions import config -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/grai-integrations/source-bigquery/src/grai_source_bigquery/loader.py b/grai-integrations/source-bigquery/src/grai_source_bigquery/loader.py index 6278e6882..a88f869ce 100644 --- a/grai-integrations/source-bigquery/src/grai_source_bigquery/loader.py +++ b/grai-integrations/source-bigquery/src/grai_source_bigquery/loader.py @@ -61,8 +61,8 @@ def __init__( ): self.namespace = get_from_env("namespace", "default") if namespace is None else namespace self.project = get_from_env("project", required=False) if project is None else project - self.dataset = get_from_env("dataset", required=False) if dataset is None else dataset - self.datasets = [self.dataset] if isinstance(self.dataset, str) else self.dataset + dataset = get_from_env("dataset", required=False) if dataset is None else dataset + self.datasets = [dataset] if isinstance(dataset, str) else dataset self.credentials = get_from_env("credentials", required=False) if credentials is None else credentials self._connection: Optional[bigquery.connector.BigqueryConnection] = None @@ -394,9 +394,7 @@ def logs(self) -> List[Any]: f' AND timestamp>="{yesterday.strftime(time_format)}"' ) - datasets = [self.dataset] if isinstance(self.dataset, str) else self.dataset - - for dataset in datasets: + for dataset in self.datasets: filter_str += ( f' AND NOT protoPayload.metadata.jobChange.job.jobStats.queryStats.referencedTables="projects/grai-demo/datasets/{dataset}/tables/INFORMATION_SCHEMA.TABLES"' f' AND NOT protoPayload.metadata.jobInsertion.job.jobStats.queryStats.referencedTables="projects/grai-demo/datasets/{dataset}/tables/INFORMATION_SCHEMA.TABLES"' diff --git a/grai-server/app/lineage/admin.py b/grai-server/app/lineage/admin.py index 9ab4d3b59..9651ad282 100755 --- a/grai-server/app/lineage/admin.py +++ b/grai-server/app/lineage/admin.py @@ -8,6 +8,20 @@ from connections.models import Connection from .models import Edge, Event, Filter, Node, Source +from connections.models import Run + + +@admin.action(description="Force delete selected sources") +def delete_sources(modeladmin, request, queryset): # pragma: no cover + sources = queryset + + for source in sources: + for connection in source.connections.all(): + Run.objects.filter(connection=connection).delete() + + connection.delete() + + queryset.delete() class EdgeInline(admin.TabularInline): @@ -180,6 +194,10 @@ class SourceAdmin(admin.ModelAdmin): ConnectionInline, ] + actions = [ + delete_sources, + ] + admin.site.register(Node, NodeAdmin) admin.site.register(Edge, EdgeAdmin)