Merge branch 'main' into saml-auth-callback

.github/workflows/py-cli-e2e-tests.yml

@@ -195,8 +195,8 @@ jobs:
  with:
  payload: |
  {
- "text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥"
- }
+ "text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\nCommit: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}" 
+ } 
  env:
  SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

.github/workflows/py-tests.yml

@@ -31,6 +31,10 @@ on:
 permissions:
  contents: read
 
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
 jobs:
  py-run-tests:
  runs-on: ubuntu-latest

bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql

@@ -24,7 +24,10 @@ CREATE INDEX data_quality_data_time_series_id_index ON data_quality_data_time_se
 -- Remove VIRTUAL status column from test_case table and remove
 -- testCaseResult state from testCase; fetch from search repo.
 ALTER TABLE test_case DROP COLUMN status;
-UPDATE test_case SET json = JSON_SET(json, '$.testCaseStatus', JSON_EXTRACT(json, '$.testCaseResult.testCaseStatus'));
+
+UPDATE test_case SET json = JSON_SET(json, '$.testCaseStatus', JSON_EXTRACT(json, '$.testCaseResult.testCaseStatus'))
+WHERE JSON_EXTRACT(json, '$.testCaseResult.testCaseStatus') IS NOT NULL;
+
 ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(json, '$.testCaseStatus'))) STORED;
 
 
@@ -33,4 +36,9 @@ UPDATE test_suite
 SET json = JSON_REMOVE(json, '$.testCaseResultSummary');
 
 UPDATE test_case
-SET json = JSON_REMOVE(json, '$.testCaseResult');
+SET json = JSON_REMOVE(json, '$.testCaseResult');
+
+-- Add Supports interrupts to SearchIndexingApplication
+UPDATE installed_apps SET json = JSON_SET(json, '$.supportsInterrupt', true) where name = 'SearchIndexingApplication'; 
+UPDATE apps_marketplace SET json = JSON_SET(json, '$.supportsInterrupt', true) where name = 'SearchIndexingApplication';
+

bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql

@@ -28,7 +28,10 @@ CREATE INDEX IF NOT EXISTS data_quality_data_time_series_id_index ON data_qual
 -- Remove VIRTUAL status column from test_case table and remove
 -- testCaseResult state from testCase; fetch from search repo.
 ALTER TABLE test_case DROP COLUMN status;
-UPDATE test_case SET json = jsonb_set(json, '{testCaseStatus}', json->'testCaseResult'->'testCaseStatus');
+
+UPDATE test_case SET json = jsonb_set(json, '{testCaseStatus}', json->'testCaseResult'->'testCaseStatus')
+WHERE json->'testCaseResult'->'testCaseStatus' IS NOT NULL;
+
 ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (json ->> 'testCaseStatus') STORED NULL;
 
 
@@ -38,3 +41,20 @@ SET json = json - 'testCaseResultSummary';
 
 UPDATE test_case
 SET json = json - 'testCaseResult';
+
+-- Add Supports interrupts to SearchIndexingApplication
+UPDATE apps_marketplace
+SET json = jsonb_set(
+ json::jsonb,
+ '{supportsInterrupt}',
+ to_jsonb(true)
+)
+where name = 'SearchIndexingApplication';
+
+UPDATE installed_apps
+SET json = jsonb_set(
+ json::jsonb,
+ '{supportsInterrupt}',
+ to_jsonb(true)
+)
+where name = 'SearchIndexingApplication';

ingestion/setup.py

@@ -299,6 +299,7 @@
  "psycopg2-binary",
  VERSIONS["geoalchemy2"],
  },
+ "mstr": {"mstr-rest-requests==0.14.1"},
  "sagemaker": {VERSIONS["boto3"]},
  "salesforce": {"simple_salesforce~=1.11"},
  "sample-data": {VERSIONS["avro"], VERSIONS["grpc-tools"]},
@@ -378,7 +379,7 @@
  "kafka-python==2.0.2",
  *plugins["pii-processor"],
  "requests==2.31.0",
- f"{DATA_DIFF['mysql']}==0.11.2",
+ f"{DATA_DIFF['mysql']}",
  *plugins["deltalake"],
  *plugins["datalake-gcs"],
  *plugins["pgspider"],

ingestion/src/_openmetadata_testutils/ometa.py

@@ -11,12 +11,14 @@
 OM_JWT = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
 
 
-def int_admin_ometa(url: str = "http://localhost:8585/api") -> OpenMetadata:
+def int_admin_ometa(
+ url: str = "http://localhost:8585/api", jwt: str = OM_JWT
+) -> OpenMetadata:
  """Initialize the ometa connection with default admin:admin creds"""
  server_config = OpenMetadataConnection(
  hostPort=url,
  authProvider=AuthProvider.openmetadata,
- securityConfig=OpenMetadataJWTClientConfig(jwtToken=CustomSecretStr(OM_JWT)),
+ securityConfig=OpenMetadataJWTClientConfig(jwtToken=CustomSecretStr(jwt)),
  )
  metadata = OpenMetadata(server_config)
  assert metadata.health_check()

ingestion/src/metadata/examples/workflows/sigma.yaml

@@ -0,0 +1,25 @@
+source:
+ type: sigma
+ serviceName: local_sigma
+ serviceConnection:
+ config:
+ type: Sigma
+ hostPort: https://api.sigmacomputing.com
+ clientId: client_id
+ clientSecret: client_secret
+ apiVersion: v2
+ sourceConfig:
+ config:
+ type: DashboardMetadata
+ lineageInformation:
+ dbServiceNames: [db_service_name]
+sink:
+ type: metadata-rest
+ config: {}
+workflowConfig:
+ loggerLevel: DEBUG # DEBUG, INFO, WARN or ERROR
+ openMetadataServerConfig:
+ hostPort: http://localhost:8585/api
+ authProvider: openmetadata
+ securityConfig:
+ jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py

@@ -13,8 +13,13 @@
 
 To be used by OpenMetadata class
 """
-from metadata.generated.schema.entity.feed.suggestion import Suggestion
+from typing import Union
+
+from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType
+from metadata.generated.schema.type import basic
+from metadata.generated.schema.type.basic import FullyQualifiedEntityName
 from metadata.ingestion.ometa.client import REST
+from metadata.ingestion.ometa.utils import model_str
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -30,12 +35,50 @@ class OMetaSuggestionsMixin:
  client: REST
 
  def update_suggestion(self, suggestion: Suggestion) -> Suggestion:
- """
- Update an existing Suggestion with new fields
- """
+ """Update an existing Suggestion with new fields"""
  resp = self.client.put(
  f"{self.get_suffix(Suggestion)}/{str(suggestion.root.id.root)}",
  data=suggestion.model_dump_json(),
  )
 
  return Suggestion(**resp)
+
+ def accept_suggestion(self, suggestion_id: Union[str, basic.Uuid]) -> None:
+ """Accept a given suggestion"""
+ self.client.put(
+ f"{self.get_suffix(Suggestion)}/{model_str(suggestion_id)}/accept",
+ )
+
+ def reject_suggestion(self, suggestion_id: Union[str, basic.Uuid]) -> None:
+ """Reject a given suggestion"""
+ self.client.put(
+ f"{self.get_suffix(Suggestion)}/{model_str(suggestion_id)}/reject",
+ )
+
+ def accept_all_suggestions(
+ self,
+ fqn: Union[str, FullyQualifiedEntityName],
+ user_id: Union[str, basic.Uuid],
+ suggestion_type: SuggestionType = SuggestionType.SuggestDescription,
+ ) -> None:
+ """Accept all suggestions"""
+ self.client.put(
+ f"{self.get_suffix(Suggestion)}/accept-all?"
+ f"userId={model_str(user_id)}&"
+ f"entityFQN={model_str(fqn)}&"
+ f"suggestionType={suggestion_type.value}",
+ )
+
+ def reject_all_suggestions(
+ self,
+ fqn: Union[str, FullyQualifiedEntityName],
+ user_id: Union[str, basic.Uuid],
+ suggestion_type: SuggestionType = SuggestionType.SuggestDescription,
+ ) -> None:
+ """Accept all suggestions"""
+ self.client.put(
+ f"{self.get_suffix(Suggestion)}/reject-all?"
+ f"userId={model_str(user_id)}&"
+ f"entityFQN={model_str(fqn)}&"
+ f"suggestionType={suggestion_type.value}",
+ )

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -19,6 +19,7 @@
 
 from pydantic import BaseModel
 
+from metadata.generated.schema.api.createBot import CreateBot
 from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import (
  CreateIngestionPipelineRequest,
 )
@@ -172,13 +173,16 @@ def get_suffix(entity: Type[T]) -> str:
 
  return route
 
- def get_module_path(self, entity: Type[T]) -> str:
+ def get_module_path(self, entity: Type[T]) -> Optional[str]:
  """
  Based on the entity, return the module path
  it is found inside generated
  """
  if issubclass(entity, CreateIngestionPipelineRequest):
  return "services.ingestionPipelines"
+ if issubclass(entity, CreateBot):
+ # Bots schemas don't live inside any subdirectory
+ return None
  return entity.__module__.split(".")[-2]
 
  def get_create_entity_type(self, entity: Type[T]) -> Type[C]:

ingestion/src/metadata/ingestion/ometa/routes.py

@@ -21,6 +21,7 @@
  CreateClassificationRequest,
 )
 from metadata.generated.schema.api.classification.createTag import CreateTagRequest
+from metadata.generated.schema.api.createBot import CreateBot
 from metadata.generated.schema.api.data.createAPICollection import (
  CreateAPICollectionRequest,
 )
@@ -213,7 +214,8 @@
  User.__name__: "/users",
  CreateUserRequest.__name__: "/users",
  AuthenticationMechanism.__name__: "/users/auth-mechanism",
- Bot.__name__: "/bots", # We won't allow bot creation from the client
+ Bot.__name__: "/bots",
+ CreateBot.__name__: "/bots",
  # Roles
  Role.__name__: "/roles",
  CreateRoleRequest.__name__: "/roles",