Fixes GEN-1260: Add Validators while creating table to escape special characters

ingestion/examples/sample_data/looker/dashboardDataModels.json

@@ -158,6 +158,51 @@
         "ordinalPosition": 5
       }
     ]
+  },
+  {
+    "name": "\"orders_view\" || \"operations_view\"",
+    "displayName": "Orders View and Operations View",
+    "description": "Orders View and Operations View from Sample Data",
+    "dataModelType":  "LookMlView",
+    "serviceType": "Looker",
+    "sql":  "view: orders {\n  sql_table_name: orders ;;\n\n  dimension: \"1. Phase I\" {\n    type: string\n    sql: ${TABLE}.status ;;\n  }\n\n  dimension: \"4. Authorized\" {\n    type: int\n    sql: ${TABLE}.amount ;;\n  }\n}",
+    "columns": [
+      {
+        "name": "0. Pre-clinical",
+        "dataType": "NUMERIC",
+        "dataTypeDisplay": "numeric",
+        "description": "Vaccine Candidates in phase: 'Pre-clinical'",
+        "ordinalPosition": 1
+      },
+      {
+        "name": "2. Phase II or Combined I/II",
+        "dataType": "NUMERIC",
+        "dataTypeDisplay": "numeric",
+        "description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'",
+        "ordinalPosition": 2
+      },
+      {
+        "name": "1. Phase I",
+        "dataType": "NUMERIC",
+        "dataTypeDisplay": "numeric",
+        "description": "Vaccine Candidates in phase: 'Phase I'",
+        "ordinalPosition": 3
+      },
+      {
+        "name": "3. Phase III",
+        "dataType": "NUMERIC",
+        "dataTypeDisplay": "numeric",
+        "description": "Vaccine Candidates in phase: 'Phase III'",
+        "ordinalPosition": 4
+      },
+      {
+        "name": "4. Authorized",
+        "dataType": "NUMERIC",
+        "dataTypeDisplay": "numeric",
+        "description": "Vaccine Candidates in phase: 'Authorize'",
+        "ordinalPosition": 5
+      }
+    ]
   }
 ]
 

ingestion/examples/sample_data/tests/testSuites.json

@@ -87,6 +87,87 @@
                         ]
                     }
                 },
+                {
+                    "name": "table_column_count_equals",
+                    "description": "test the number of column in table",
+                    "entityLink": "<#E::table::sample_data.ecommerce_db.shopify.dim___reserved__colon____reserved__arrow__address>",
+                    "testDefinitionName": "tableColumnCountToEqual",
+                    "parameterValues": [
+                        {
+                            "name": "columnCount",
+                            "value": "10"
+                        }
+                    ],
+                    "resolutions": {
+                        "sequenceOne": [
+                            {
+                                "testCaseResolutionStatusType": "Ack",
+                                "severity": "Severity1"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Assigned",
+                                "severity": "Severity1",
+                                "assignee": "aaron_johnson0"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Resolved",
+                                "severity": "Severity1",
+                                "resolver": "aaron_johnson0"
+                            }
+                        ],
+                        "sequenceTwo": [
+                            {
+                                "testCaseResolutionStatusType": "New",
+                                "severity": "Severity1"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Ack",
+                                "severity": "Severity1"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Assigned",
+                                "severity": "Severity1",
+                                "assignee": "christopher_campbell7"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Resolved",
+                                "severity": "Severity1",
+                                "resolver": "christopher_campbell7"
+                            }
+                        ],
+                        "sequenceThree": [
+                            {
+                                "testCaseResolutionStatusType": "New",
+                                "severity": "Severity3"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Ack",
+                                "severity": "Severity3"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Assigned",
+                                "severity": "Severity3",
+                                "assignee": "christopher_campbell7"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Assigned",
+                                "severity": "Severity3",
+                                "assignee": "aaron_johnson0"
+                            },
+                            {
+                                "testCaseResolutionStatusType": "Resolved",
+                                "severity": "Severity3",
+                                "resolver": "aaron_johnson0"
+                            }
+                        ],
+                        "sequenceFour": [
+                            {
+                                "testCaseResolutionStatusType": "New",
+                                "severity": "Severity5"
+                            }
+                        ]
+                    }
+                },
                 {
                     "name": "table_column_count_between",
                     "description": "test the number of column in table is between x and y",

ingestion/src/metadata/config/common.py

@@ -19,7 +19,9 @@
 from typing import IO, Any, Optional
 
 import yaml
-from pydantic import BaseModel, ConfigDict
+from pydantic import ConfigDict
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 
 class ConfigModel(BaseModel):

ingestion/src/metadata/data_quality/api/models.py

@@ -18,13 +18,14 @@
 
 from typing import List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
 
 from metadata.config.common import ConfigModel
 from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.tests.basic import TestCaseResult
 from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 
 class TestCaseDefinition(ConfigModel):

ingestion/src/metadata/ingestion/models/custom_basemodel_validation.py

@@ -0,0 +1,80 @@
+#  Copyright 2022 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Validation logic for Custom Pydantic BaseModel
+"""
+
+import logging
+
+logger = logging.getLogger("metadata")
+
+
+RESTRICTED_KEYWORDS = ["::", ">"]
+RESERVED_COLON_KEYWORD = "__reserved__colon__"
+RESERVED_ARROW_KEYWORD = "__reserved__arrow__"
+
+CREATE_ADJACENT_MODELS = {"ProfilerResponse", "SampleData"}
+NAME_FIELDS = {"EntityName", "str", "ColumnName", "TableData"}
+FETCH_MODELS = {"Table", "CustomColumnName"}
+FIELD_NAMES = {"name", "columns", "root"}
+
+
+def revert_separators(value):
+    return value.replace(RESERVED_COLON_KEYWORD, "::").replace(
+        RESERVED_ARROW_KEYWORD, ">"
+    )
+
+
+def replace_separators(value):
+    return value.replace("::", RESERVED_COLON_KEYWORD).replace(
+        ">", RESERVED_ARROW_KEYWORD
+    )
+
+
+def validate_name_and_transform(values, modification_method, field_name: str = None):
+    """
+    Validate the name and transform it if needed.
+    """
+    if isinstance(values, str) and field_name in FIELD_NAMES:
+        values = modification_method(values)
+    elif (
+        hasattr(values, "root")
+        and isinstance(values.root, str)
+        and field_name in FIELD_NAMES
+    ):
+        values.root = modification_method(values.root)
+    elif hasattr(values, "model_fields"):
+        for key in type(values).model_fields.keys():
+            if getattr(values, key):
+                if getattr(values, key).__class__.__name__ in NAME_FIELDS:
+                    setattr(
+                        values,
+                        key,
+                        validate_name_and_transform(
+                            getattr(values, key),
+                            modification_method=modification_method,
+                            field_name=key,
+                        ),
+                    )
+                elif isinstance(getattr(values, key), list):
+                    setattr(
+                        values,
+                        key,
+                        [
+                            validate_name_and_transform(
+                                item,
+                                modification_method=modification_method,
+                                field_name=key,
+                            )
+                            for item in getattr(values, key)
+                        ],
+                    )
+    return values

ingestion/src/metadata/ingestion/models/custom_pydantic.py

@@ -20,11 +20,19 @@
 from typing import Any, Dict, Literal, Optional, Union
 
 from pydantic import BaseModel as PydanticBaseModel
-from pydantic import PlainSerializer
+from pydantic import PlainSerializer, model_validator
 from pydantic.main import IncEx
 from pydantic.types import SecretStr
 from typing_extensions import Annotated
 
+from metadata.ingestion.models.custom_basemodel_validation import (
+    CREATE_ADJACENT_MODELS,
+    FETCH_MODELS,
+    replace_separators,
+    revert_separators,
+    validate_name_and_transform,
+)
+
 logger = logging.getLogger("metadata")
 
 SECRET = "secret:"
@@ -37,6 +45,30 @@ class BaseModel(PydanticBaseModel):
     Specified as `--base-class BASE_CLASS` in the generator.
     """
 
+    @model_validator(mode="after")
+    @classmethod
+    def parse_name(cls, values):  # pylint: disable=inconsistent-return-statements
+        """
+        Primary entry point to process values based on their class.
+        """
+
+        if not values:
+            return
+
+        try:
+
+            if cls.__name__ in CREATE_ADJACENT_MODELS or cls.__name__.startswith(
+                "Create"
+            ):
+                values = validate_name_and_transform(values, replace_separators)
+            elif cls.__name__ in FETCH_MODELS:
+                values = validate_name_and_transform(values, revert_separators)
+
+        except Exception as exc:
+            logger.warning("Exception while parsing Basemodel: %s", exc)
+            raise exc
+        return values
+
     def model_dump_json(  # pylint: disable=too-many-arguments
         self,
         *,

ingestion/src/metadata/ingestion/source/database/sample_data.py

@@ -136,6 +136,7 @@
     InvalidSchemaTypeException,
     schema_parser_config_registry,
 )
+from metadata.profiler.api.models import ProfilerResponse, SampleData
 from metadata.utils import entity_link, fqn
 from metadata.utils.constants import UTF_8
 from metadata.utils.fqn import FQN_SEPARATOR
@@ -898,10 +899,24 @@ def ingest_tables(self) -> Iterable[Either[Entity]]:
 
                 self.metadata.ingest_table_sample_data(
                     table_entity,
-                    TableData(
-                        rows=table["sampleData"]["rows"],
-                        columns=table["sampleData"]["columns"],
-                    ),
+                    ProfilerResponse(
+                        table=table_entity,
+                        profile=CreateTableProfileRequest(
+                            tableProfile=TableProfile(
+                                timestamp=Timestamp(
+                                    int(datetime.now().timestamp() * 1000)
+                                ),
+                                columnCount=1.0,
+                                rowCount=3.0,
+                            )
+                        ),
+                        sample_data=SampleData(
+                            data=TableData(
+                                rows=table["sampleData"]["rows"],
+                                columns=table["sampleData"]["columns"],
+                            )
+                        ),
+                    ).sample_data.data,
                 )
 
             if table.get("customMetrics"):
@@ -1331,12 +1346,14 @@ def ingest_mlmodels(self) -> Iterable[Either[CreateMlModelRequest]]:
                     description=model["description"],
                     algorithm=model["algorithm"],
                     dashboard=dashboard.fullyQualifiedName.root,
-                    mlStore=MlStore(
-                        storage=model["mlStore"]["storage"],
-                        imageRepository=model["mlStore"]["imageRepository"],
-                    )
-                    if model.get("mlStore")
-                    else None,
+                    mlStore=(
+                        MlStore(
+                            storage=model["mlStore"]["storage"],
+                            imageRepository=model["mlStore"]["imageRepository"],
+                        )
+                        if model.get("mlStore")
+                        else None
+                    ),
                     server=model.get("server"),
                     target=model.get("target"),
                     mlFeatures=self.get_ml_features(model),
@@ -1375,9 +1392,11 @@ def ingest_containers(self) -> Iterable[Either[CreateContainerRequest]]:
                     name=container["name"],
                     displayName=container["displayName"],
                     description=container["description"],
-                    parent=EntityReference(id=parent_container.id, type="container")
-                    if parent_container_fqn
-                    else None,
+                    parent=(
+                        EntityReference(id=parent_container.id, type="container")
+                        if parent_container_fqn
+                        else None
+                    ),
                     prefix=container["prefix"],
                     dataModel=container.get("dataModel"),
                     numberOfObjects=container.get("numberOfObjects"),
@@ -1415,11 +1434,13 @@ def ingest_containers(self) -> Iterable[Either[CreateContainerRequest]]:
                     yield Either(
                         right=CreateContainerRequest(
                             name=name,
-                            parent=EntityReference(
-                                id=parent_container.id, type="container"
-                            )
-                            if parent_container
-                            else None,
+                            parent=(
+                                EntityReference(
+                                    id=parent_container.id, type="container"
+                                )
+                                if parent_container
+                                else None
+                            ),
                             service=self.storage_service.fullyQualifiedName,
                         )
                     )