-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
82 changed files
with
2,670 additions
and
1,160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| source: | ||
| type: sigma | ||
| serviceName: local_sigma | ||
| serviceConnection: | ||
| config: | ||
| type: Sigma | ||
| hostPort: https://api.sigmacomputing.com | ||
| clientId: client_id | ||
| clientSecret: client_secret | ||
| apiVersion: v2 | ||
| sourceConfig: | ||
| config: | ||
| type: DashboardMetadata | ||
| lineageInformation: | ||
| dbServiceNames: [db_service_name] | ||
| sink: | ||
| type: metadata-rest | ||
| config: {} | ||
| workflowConfig: | ||
| loggerLevel: DEBUG # DEBUG, INFO, WARN or ERROR | ||
| openMetadataServerConfig: | ||
| hostPort: http://localhost:8585/api | ||
| authProvider: openmetadata | ||
| securityConfig: | ||
| jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
187 changes: 187 additions & 0 deletions
187
ingestion/src/metadata/ingestion/source/dashboard/sigma/client.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,187 @@ | ||
| # Copyright 2021 Collate | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| """ | ||
| REST Auth & Client for Sigma | ||
| """ | ||
|
|
||
| import traceback | ||
| from base64 import b64encode | ||
| from typing import List, Optional, Tuple | ||
|
|
||
| from metadata.generated.schema.entity.services.connections.dashboard.sigmaConnection import ( | ||
| SigmaConnection, | ||
| ) | ||
| from metadata.ingestion.ometa.client import REST, ClientConfig | ||
| from metadata.ingestion.source.dashboard.sigma.models import ( | ||
| AuthToken, | ||
| EdgeSource, | ||
| EdgeSourceResponse, | ||
| Elements, | ||
| ElementsResponse, | ||
| NodeDetails, | ||
| OwnerDetails, | ||
| Workbook, | ||
| WorkbookDetails, | ||
| WorkBookPageResponse, | ||
| WorkBookResponseDetails, | ||
| ) | ||
| from metadata.utils.constants import AUTHORIZATION_HEADER, UTF_8 | ||
| from metadata.utils.helpers import clean_uri | ||
| from metadata.utils.logger import utils_logger | ||
|
|
||
| logger = utils_logger() | ||
|
|
||
| HEADERS = { | ||
| "accept": "application/json", | ||
| "Content-type": "application/x-www-form-urlencoded", | ||
| } | ||
|
|
||
| TOKEN_PAYLOAD = {"grant_type": "client_credentials"} | ||
|
|
||
|
|
||
| class SigmaApiClient: | ||
| """ | ||
| REST Auth & Client for Sigma | ||
| """ | ||
|
|
||
| client: REST | ||
|
|
||
| def __init__(self, config: SigmaConnection): | ||
| self.config = config | ||
| token_api_key = str( | ||
| b64encode( | ||
| f"{self.config.clientId}:{self.config.clientSecret.get_secret_value()}".encode( | ||
| UTF_8 | ||
| ) | ||
| ).decode(UTF_8) | ||
| ) | ||
|
|
||
| token_config = ClientConfig( | ||
| base_url=clean_uri(config.hostPort), | ||
| api_version=config.apiVersion, | ||
| auth_header=AUTHORIZATION_HEADER, | ||
| extra_headers=HEADERS, | ||
| auth_token=lambda: (token_api_key, 0), | ||
| auth_token_mode="Basic", | ||
| ) | ||
|
|
||
| self.token_client = REST(token_config) | ||
|
|
||
| client_config = ClientConfig( | ||
| base_url=clean_uri(config.hostPort), | ||
| api_version=config.apiVersion, | ||
| auth_token=self.get_auth_token, | ||
| auth_header=AUTHORIZATION_HEADER, | ||
| ) | ||
|
|
||
| self.client = REST(client_config) | ||
|
|
||
| def get_auth_token(self) -> Tuple[str, int]: | ||
| """ | ||
| generate auth token | ||
| Returns: | ||
| Tuple[str, int]: A tuple containing the access_token (str) and expires_in (int) | ||
| """ | ||
| result = AuthToken.model_validate( | ||
| self.token_client.post("/auth/token", data=TOKEN_PAYLOAD) | ||
| ) | ||
| return result.access_token, result.expires_in | ||
|
|
||
| def get_dashboards(self) -> Optional[List[Workbook]]: | ||
| """ | ||
| method to fetch dashboards from api | ||
| """ | ||
| result = WorkBookResponseDetails.model_validate(self.client.get("/workbooks")) | ||
| if result: | ||
| return result.entries | ||
|
|
||
| def get_dashboard_detail(self, workbook_id: str) -> Optional[WorkbookDetails]: | ||
| """ | ||
| method to fetch dashboard details from api | ||
| """ | ||
| try: | ||
| result = WorkbookDetails.model_validate( | ||
| self.client.get(f"/workbooks/{workbook_id}") | ||
| ) | ||
| if result: | ||
| return result | ||
| except Exception as exc: # pylint: disable=broad-except | ||
| logger.debug(traceback.format_exc()) | ||
| logger.error( | ||
| f"Error fetching Dashboard details for for workbook {workbook_id}: {exc}" | ||
| ) | ||
| return None | ||
|
|
||
| def get_owner_detail(self, owner_id: str) -> Optional[OwnerDetails]: | ||
| """ | ||
| method to fetch dashboard owner details from api | ||
| """ | ||
| try: | ||
| result = OwnerDetails.model_validate( | ||
| self.client.get(f"/members/{owner_id}") | ||
| ) | ||
| if result: | ||
| return result | ||
| except Exception as exc: # pylint: disable=broad-except | ||
| logger.debug(traceback.format_exc()) | ||
| logger.warning(f"Failed to fetch owner details for owner {owner_id}: {exc}") | ||
| return None | ||
|
|
||
| def get_chart_details(self, workbook_id: str) -> Optional[List[Elements]]: | ||
| """ | ||
| method to fetch dashboards chart details from api | ||
| """ | ||
| try: | ||
| elements_list = [] | ||
| pages = WorkBookPageResponse.model_validate( | ||
| self.client.get(f"/workbooks/{workbook_id}/pages") | ||
| ) | ||
| for page in pages.entries: | ||
| elements = ElementsResponse.model_validate( | ||
| self.client.get( | ||
| f"/workbooks/{workbook_id}/pages/{page.pageId}/elements" | ||
| ) | ||
| ) | ||
| elements_list.extend(elements.entries or []) | ||
| return elements_list | ||
| except Exception as exc: # pylint: disable=broad-except | ||
| logger.debug(traceback.format_exc()) | ||
| logger.warning( | ||
| f"Failed to fetch chart details for workbook {workbook_id}: {exc}" | ||
| ) | ||
| return None | ||
|
|
||
| def get_lineage_details( | ||
| self, workbook_id: str, element_id: str | ||
| ) -> Optional[List[EdgeSource]]: | ||
| """ | ||
| method to fetch dashboards lineage details from api | ||
| """ | ||
| try: | ||
| source_nodes = [] | ||
| edges_response = EdgeSourceResponse.model_validate( | ||
| self.client.get( | ||
| f"/workbooks/{workbook_id}/lineage/elements/{element_id}" | ||
| ) | ||
| ) | ||
| for node in edges_response.edges: | ||
| if node.node_id: | ||
| node_details = NodeDetails.model_validate( | ||
| self.client.get(f"/files/{node.node_id}") | ||
| ) | ||
| source_nodes.append(node_details) | ||
| return source_nodes | ||
| except Exception as exc: # pylint: disable=broad-except | ||
| logger.debug(traceback.format_exc()) | ||
| logger.warning( | ||
| f"Failed to fetch lineage details for workbook {workbook_id}: {exc}" | ||
| ) | ||
| return None |
Oops, something went wrong.