Merge remote-tracking branch 'upstream/main' into MINOR-SecretUtils-E…

…rror-log

bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql

@@ -1773,3 +1773,27 @@ SET json = JSON_SET(
     )
 )
 WHERE serviceType = 'DBTCloud';
+
+-- Update serviceType in dashboard_entity table
+UPDATE dashboard_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
+
+-- Update serviceType in dashboard_service_entity table
+UPDATE dashboard_service_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
+
+UPDATE dashboard_service_entity
+SET json = JSON_SET(json, '$.connection.config.type', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.connection.config.type')) = 'Mstr';
+
+-- Update serviceType in dashboard_data_model_entity table
+UPDATE dashboard_data_model_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
+
+-- Update serviceType in chart_entity table
+UPDATE chart_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';

bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql

@@ -1756,3 +1756,27 @@ and servicetype = 'DBTCloud';
 UPDATE pipeline_service_entity
 SET json = jsonb_set(json, '{connection, config, projectIds}', '[]', true)
 WHERE servicetype = 'DBTCloud';
+
+-- Update serviceType in dashboard_entity table
+UPDATE dashboard_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
+
+-- Update serviceType in dashboard_service_entity table
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{connection,config,type}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'connection', 'config', 'type') = 'Mstr';
+
+-- Update serviceType in dashboard_data_model_entity table
+UPDATE dashboard_data_model_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
+
+-- Update serviceType in chart_entity table
+UPDATE chart_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';

ingestion/src/metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py

@@ -104,6 +104,7 @@ def _create_runner(self) -> QueryRunner:
             QueryRunner(
                 session=self.session,
                 dataset=self.dataset,
+                raw_dataset=self.sampler.raw_dataset,
                 partition_details=self.table_partition_config,
                 profile_sample_query=self.table_sample_query,
             )

ingestion/src/metadata/data_quality/runner/base_test_suite_source.py

@@ -49,9 +49,10 @@ def __init__(
     ):
         self.validator_builder_class = ValidatorBuilder
         self._interface = None
-        self._interface_type: str = config.source.type.lower()
         self.entity = entity
         self.service_conn_config = self._copy_service_config(config, self.entity.database)  # type: ignore
+        self._interface_type: str = self.service_conn_config.type.value.lower()
+
         self.source_config = TestSuitePipeline.model_validate(
             config.source.sourceConfig.config
         )

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py

@@ -40,7 +40,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py

@@ -38,7 +38,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py

@@ -42,7 +42,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column, **kwargs) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py

@@ -37,7 +37,7 @@ class ColumnValuesToBeAtExpectedLocationValidator(
     def _fetch_data(self, columns: List[str]) -> Iterator:
         """Fetch data from the runner object"""
         self.runner = cast(QueryRunner, self.runner)
-        inspection = inspect(self.runner.table)
+        inspection = inspect(self.runner.dataset)
         table_columns: List[Column] = inspection.c if inspection is not None else []
         cols = [col for col in table_columns if col.name in columns]
         for col in cols:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column, **kwargs) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py

@@ -39,7 +39,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column, **kwargs) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py

@@ -42,7 +42,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py

@@ -17,7 +17,6 @@
 
 from sqlalchemy import Column, inspect
 from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.orm.util import AliasedClass
 
 from metadata.data_quality.validations.column.base.columnValuesToBeUnique import (
     BaseColumnValuesToBeUniqueValidator,
@@ -41,7 +40,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:
@@ -53,12 +52,7 @@ def _run_results(self, metric: Metrics, column: Column) -> Optional[int]:
         """
         count = Metrics.COUNT.value(column).fn()
         unique_count = Metrics.UNIQUE_COUNT.value(column).query(
-            sample=self.runner._sample  # pylint: disable=protected-access
-            if isinstance(
-                self.runner._sample,  # pylint: disable=protected-access
-                AliasedClass,
-            )
-            else self.runner.table,
+            sample=self.runner.dataset,
             session=self.runner._session,  # pylint: disable=protected-access
         )  # type: ignore
 

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py

@@ -43,7 +43,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(

ingestion/src/metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py

@@ -43,7 +43,7 @@ def _get_column_name(self) -> Column:
         """
         return self.get_column_name(
             self.test_case.entityLink.root,
-            inspect(self.runner.table).c,
+            inspect(self.runner.dataset).c,
         )
 
     def _run_results(self, metric: Metrics, column: Column, **kwargs) -> Optional[int]:

ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py

@@ -13,7 +13,7 @@
 Validator for table row inserted count to be between test case
 """
 
-from sqlalchemy import Column, text
+from sqlalchemy import Column, inspect, text
 
 from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
     SQAValidatorMixin,
@@ -52,7 +52,7 @@ def _run_results(self, column_name: str, range_type: str, range_interval: int):
         date_or_datetime_fn = dispatch_to_date_or_datetime(
             range_interval,
             text(range_type),
-            get_partition_col_type(column_name.name, self.runner.table.c),  # type: ignore
+            get_partition_col_type(column_name.name, inspect(self.runner.dataset).c),  # type: ignore
         )
 
         return dict(

ingestion/src/metadata/examples/workflows/mstr.yaml → ingestion/src/metadata/examples/workflows/microstrategy.yaml

@@ -1,13 +1,14 @@
 source:
-  type: mstr
+  type: microstrategy
   serviceName: test
   serviceConnection:
     config:
-      type: Mstr
+      type: MicroStrategy
       username: username
       password: password
       hostPort: http://hostPort
       projectName: project
+      loginMode: "8"
   sourceConfig:
     config:
       type: DashboardMetadata

ingestion/src/metadata/ingestion/source/dashboard/mstr/client.py → ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py

@@ -9,19 +9,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """
-REST Auth & Client for Mstr
+REST Auth & Client for MicroStrategy
 """
 import traceback
 from typing import List, Optional
 
 import requests
 
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.ingestion.connections.test_connections import SourceConnectionException
 from metadata.ingestion.ometa.client import REST, ClientConfig
-from metadata.ingestion.source.dashboard.mstr.models import (
+from metadata.ingestion.source.dashboard.microstrategy.models import (
     AuthHeaderCookie,
     MstrDashboard,
     MstrDashboardDetails,
@@ -37,30 +37,29 @@
 logger = ingestion_logger()
 
 API_VERSION = "MicroStrategyLibrary/api"
-LOGIN_MODE_GUEST = 8
 APPLICATION_TYPE = 35
 
 
-class MSTRClient:
+class MicroStrategyClient:
     """
     Client Handling API communication with Metabase
     """
 
     def _get_base_url(self, path=None):
         if not path:
-            return f"{clean_uri(self.config.hostPort)}/{API_VERSION}"
-        return f"{clean_uri(self.config.hostPort)}/{API_VERSION}/{path}"
+            return f"{clean_uri(str(self.config.hostPort))}/{API_VERSION}"
+        return f"{clean_uri(str(self.config.hostPort))}/{API_VERSION}/{path}"
 
     def __init__(
         self,
-        config: MstrConnection,
+        config: MicroStrategyConnection,
     ):
         self.config = config
 
         self.auth_params: AuthHeaderCookie = self._get_auth_header_and_cookies()
 
         client_config = ClientConfig(
-            base_url=clean_uri(config.hostPort),
+            base_url=clean_uri(str(self.config.hostPort)),
             api_version=API_VERSION,
             extra_headers=self.auth_params.auth_header,
             allow_redirects=True,
@@ -81,7 +80,7 @@ def _get_auth_header_and_cookies(self) -> Optional[AuthHeaderCookie]:
             data = {
                 "username": self.config.username,
                 "password": self.config.password.get_secret_value(),
-                "loginMode": LOGIN_MODE_GUEST,
+                "loginMode": self.config.loginMode,
                 "applicationType": APPLICATION_TYPE,
             }
             response = requests.post(

ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py → ingestion/src/metadata/ingestion/source/dashboard/microstrategy/connection.py

@@ -17,31 +17,29 @@
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.dashboard.mstr.client import MSTRClient
-from metadata.utils.constants import THREE_MIN
+from metadata.ingestion.source.dashboard.microstrategy.client import MicroStrategyClient
 
 
-def get_connection(connection: MstrConnection) -> MSTRClient:
+def get_connection(connection: MicroStrategyConnection) -> MicroStrategyClient:
     """
     Create connection
     """
-    return MSTRClient(connection)
+    return MicroStrategyClient(connection)
 
 
 def test_connection(
     metadata: OpenMetadata,
-    client: MSTRClient,
-    service_connection: MstrConnection,
+    client: MicroStrategyClient,
+    service_connection: MicroStrategyConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: Optional[int] = THREE_MIN,
 ) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
@@ -55,5 +53,4 @@ def test_connection(
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
-        timeout_seconds=timeout_seconds,
     )