diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py index 70d3a890ba75..8a785cd739d2 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py @@ -13,6 +13,7 @@ Source connection handler """ import os +from datetime import datetime from functools import partial from typing import Optional @@ -122,7 +123,8 @@ def test_connection_inner(engine): test_query, engine=engine, statement=BIGQUERY_TEST_STATEMENT.format( - region=service_connection.usageLocation + region=service_connection.usageLocation, + creation_date=datetime.now().strftime("%Y-%m-%d"), ), ), } diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 17500f19367e..b88af7a55a6c 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -321,7 +321,7 @@ def get_schema_description(self, schema_name: str) -> Optional[str]: ) query_result = [result.schema_description for result in query_resp.result()] - return query_result[0] + return fqn.unquote_name(query_result[0]) except IndexError: logger.debug(f"No dataset description found for {schema_name}") except Exception as err: diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/queries.py b/ingestion/src/metadata/ingestion/source/database/bigquery/queries.py index 6b80b773846c..8d16af422c97 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/queries.py @@ -38,7 +38,8 @@ ) BIGQUERY_TEST_STATEMENT = textwrap.dedent( - """SELECT query FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT limit 1""" + """SELECT query FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT + where creation_time > '{creation_date}' limit 1""" ) @@ -81,7 +82,7 @@ user_email as user_name FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT WHERE statement_type = 'SCRIPT' - AND start_time >= '{start_date}' + AND creation_time >= '{start_date}' AND job_type = "QUERY" AND state = "DONE" AND error_result is NULL @@ -102,7 +103,7 @@ WHERE statement_type <> 'SCRIPT' AND query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%' - AND start_time >= '{start_date}' + AND creation_time >= '{start_date}' AND job_type = "QUERY" AND state = "DONE" AND error_result is NULL diff --git a/ingestion/src/metadata/utils/fqn.py b/ingestion/src/metadata/utils/fqn.py index a40187042f58..669db098baa9 100644 --- a/ingestion/src/metadata/utils/fqn.py +++ b/ingestion/src/metadata/utils/fqn.py @@ -94,7 +94,7 @@ def _build(*args, quote: bool = True) -> str: def unquote_name(name: str) -> str: - return name[1:-1] if name is not None and '"' in name else name + return name[1:-1] if name and name[0] == '"' and name[-1] == '"' else name def quote_name(name: str) -> str: