fix: tests; address comments and add more tests

ingestion/src/metadata/data_quality/processor/test_case_runner.py

@@ -344,7 +344,6 @@ def filter_incompatible_test_cases(
                 result.append(tc)
                 continue
             column_name = entity_link.get_decoded_column(tc.entityLink.root)
-
             column = next(c for c in table.columns if c.name.root == column_name)
 
             if column.dataType not in test_definition.supportedDataTypes:

ingestion/src/metadata/ingestion/models/custom_basemodel_validation.py

@@ -45,7 +45,7 @@ def validate_name_and_transform(values, modification_method):
     try:
         if isinstance(values, str):
             values = modification_method(values)
-        elif hasattr(values, "root"):
+        elif hasattr(values, "root") and isinstance(values.root, str):
             values.root = modification_method(values.root)
         elif hasattr(values, "model_fields"):
             for key in type(values).model_fields.keys():

ingestion/src/metadata/ingestion/models/custom_pydantic.py

@@ -55,10 +55,18 @@ def parse_name(cls, values):  # pylint: disable=inconsistent-return-statements
         if not values:
             return
 
-        if cls.__name__ in CREATE_ADJACENT_MODELS or cls.__name__.startswith("Create"):
-            values = validate_name_and_transform(values, replace_separators)
-        elif cls.__name__ in FETCH_MODELS:
-            values = validate_name_and_transform(values, revert_separators)
+        try:
+
+            if cls.__name__ in CREATE_ADJACENT_MODELS or cls.__name__.startswith(
+                "Create"
+            ):
+                values = validate_name_and_transform(values, replace_separators)
+            elif cls.__name__ in FETCH_MODELS:
+                values = validate_name_and_transform(values, revert_separators)
+
+        except Exception as exc:
+            logger.warning("Exception while parsing Basemodel: %s", exc)
+            raise exc
         return values
 
     def model_dump_json(  # pylint: disable=too-many-arguments

ingestion/tests/integration/alationsink/test_alationsink.py

@@ -107,7 +107,7 @@ def mock_list_connectors():
 EXPECTED_TABLES = [
     CreateTableRequest(
         key="34.shopify.dim_::>address",
-        title="dim_address",
+        title="dim_::>address",
         description="This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.",
         table_type="TABLE",
         sql=None,

ingestion/tests/unit/models/test_custom_pydantic.py

@@ -0,0 +1,145 @@
+# %%
+
+
+import uuid
+
+from metadata.generated.schema.api.data.createTable import CreateTableRequest
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    ColumnName,
+    DataType,
+    Table,
+    TableConstraint,
+)
+from metadata.generated.schema.type.basic import (
+    EntityExtension,
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+
+
+def test_replace_separator():
+    create_request_replace_separator = CreateTableRequest(
+        name=EntityName("Sales::>Territory"),
+        displayName="SalesTerritory",
+        description=Markdown(root="Sales territory lookup table."),
+        tableType="Regular",
+        columns=[
+            Column(
+                name=ColumnName(root="Sales::Last>Year"),
+                displayName="SalesLastYear",
+                dataType="NUMBER",
+                arrayDataType=None,
+                dataLength=None,
+                precision=None,
+                scale=None,
+                dataTypeDisplay="NUMBER",
+                description=Markdown(root="Sales total of previous year."),
+                fullyQualifiedName=None,
+                tags=None,
+                constraint="NOT_NULL",
+                ordinalPosition=7,
+                jsonSchema=None,
+                children=None,
+                profile=None,
+                customMetrics=None,
+            ),
+            Column(
+                name=ColumnName(root="Bonus"),
+                displayName="Bonus",
+                dataType="NUMBER",
+                arrayDataType=None,
+                dataLength=None,
+                precision=None,
+                scale=None,
+                dataTypeDisplay="NUMBER",
+                description=Markdown(root="Bonus due if quota is met."),
+                fullyQualifiedName=None,
+                tags=None,
+                constraint="NOT_NULL",
+                ordinalPosition=4,
+                jsonSchema=None,
+                children=None,
+                profile=None,
+                customMetrics=None,
+            ),
+            Column(
+                name=ColumnName(root="ModifiedDate"),
+                displayName="ModifiedDate",
+                dataType="DATETIME",
+                arrayDataType=None,
+                dataLength=None,
+                precision=None,
+                scale=None,
+                dataTypeDisplay="DATETIME",
+                description=Markdown(root="Date and time the record was last updated."),
+                fullyQualifiedName=None,
+                tags=None,
+                constraint="NOT_NULL",
+                ordinalPosition=9,
+                jsonSchema=None,
+                children=None,
+                profile=None,
+                customMetrics=None,
+            ),
+        ],
+        dataModel=None,
+        tableConstraints=[
+            TableConstraint(constraintType="PRIMARY_KEY", columns=["Sales::Last>Year"])
+        ],
+        tablePartition=None,
+        tableProfilerConfig=None,
+        owners=None,
+        databaseSchema=FullyQualifiedEntityName(
+            root='New Gyro 360.New Gyro 360."AdventureWorks2017.HumanResources"'
+        ),
+        tags=None,
+        schemaDefinition=None,
+        retentionPeriod=None,
+        extension=EntityExtension(
+            root={
+                "DataQuality": '<div><p><b>Last evaluation:</b> 07/24/2023<br><b>Interval: </b>30 days <br><b>Next run:</b> 08/23/2023, 10:44:20<br><b>Measurement unit:</b> percent [%]</p><br><table><tbody><tr><th>Metric</th><th>Target</th><th>Latest result</th></tr><tr><td><p class="text-success">Completeness</p></td><td>90%</td><td><div class="bar fabric" style="width: 100%;"><strong>100%</strong></div></td></tr><tr><td><p class="text-success">Integrity</p></td><td>90%</td><td><div class="bar fabric" style="width: 100%;"><strong>100%</strong></div></td></tr><tr><td><p class="text-warning">Timeliness</p></td><td>90%</td><td><div class="bar fabric" style="width: 25%;"><strong>25%</strong></div></td></tr><tr><td><p class="text-warning">Uniqueness</p></td><td>90%</td><td><div class="bar fabric" style="width: 60%;"><strong>60%</strong></div></td></tr><tr><td><p class="text-success">Validity</p></td><td>90%</td><td><div class="bar fabric" style="width: 100%;"><strong>100%</strong></div></td></tr></tbody></table><h3>Overall score of the table is: 77%</h3><hr style="border-width: 5px;"></div>'
+            }
+        ),
+        sourceUrl=None,
+        domain=None,
+        dataProducts=None,
+        fileFormat=None,
+        lifeCycle=None,
+        sourceHash=None,
+    )
+
+    assert (
+        create_request_replace_separator.name.root
+        == "Sales__reserved__colon____reserved__arrow__Territory"
+    )
+    assert (
+        create_request_replace_separator.columns[0].name.root
+        == "Sales__reserved__colon__Last__reserved__arrow__Year"
+    )
+    assert (
+        create_request_replace_separator.tableConstraints[0].columns[0]
+        == "Sales__reserved__colon__Last__reserved__arrow__Year"
+    )
+
+
+def test_revert_separator():
+
+    fetch_response_revert_separator = Table(
+        id=uuid.uuid4(),
+        name="test__reserved__colon__table",
+        databaseSchema=EntityReference(id=uuid.uuid4(), type="databaseSchema"),
+        fullyQualifiedName="test-service-table.test-db.test-schema.test",
+        columns=[Column(name="id", dataType=DataType.BIGINT)],
+    )
+    fetch_response_revert_separator_2 = Table(
+        id=uuid.uuid4(),
+        name="test__reserved__colon__table__reserved__arrow__",
+        databaseSchema=EntityReference(id=uuid.uuid4(), type="databaseSchema"),
+        fullyQualifiedName="test-service-table.test-db.test-schema.test",
+        columns=[Column(name="id", dataType=DataType.BIGINT)],
+    )
+    assert fetch_response_revert_separator.name.root == "test::table"
+    assert fetch_response_revert_separator_2.name.root == "test::table>"