Merge branch 'main' into version-update

bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql

@@ -21,3 +21,7 @@ CREATE TABLE IF NOT EXISTS data_product_entity (
  PRIMARY KEY (id),
  UNIQUE (fqnHash)
 );
+
+UPDATE dbservice_entity
+SET json = JSON_REPLACE(json, '$.connection.config.scheme', 'hive')
+WHERE JSON_EXTRACT(json, '$.connection.config.scheme') IN ('impala', 'impala4');

bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql

@@ -21,3 +21,7 @@ CREATE TABLE IF NOT EXISTS data_product_entity (
  PRIMARY KEY (id),
  UNIQUE (fqnHash)
 );
+
+update dbservice_entity
+set json = jsonb_set(json::jsonb, '{connection,config,scheme}', '"hive"')
+where json#>>'{connection,config,scheme}' in ('impala', 'impala4');

ingestion/src/metadata/ingestion/source/database/hive/metadata.py

@@ -15,7 +15,6 @@
 import re
 from typing import Tuple
 
-from impala.sqlalchemy import ImpalaDialect
 from pyhive.sqlalchemy_hive import HiveDialect, _type_map
 from sqlalchemy import types, util
 from sqlalchemy.engine import reflection
@@ -32,7 +31,6 @@
 from metadata.ingestion.api.source import InvalidSourceException
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
 from metadata.ingestion.source.database.hive.queries import HIVE_GET_COMMENTS
-from metadata.profiler.orm.registry import Dialects
 
 complex_data_types = ["struct", "map", "array", "union"]
 
@@ -182,124 +180,6 @@ def get_table_comment( # pylint: disable=unused-argument
  return {"text": None}
 
 
-def get_impala_table_or_view_names(connection, schema=None, target_type="table"):
- """
- Depending on the targetType returns either the Views or Tables
- since they share the same method for getting their names.
- """
- query = "show tables"
- if schema:
- query += " IN " + schema
-
- cursor = connection.execute(query)
- results = cursor.fetchall()
- tables_and_views = [result[0] for result in results]
-
- retvalue = []
-
- for table_view in tables_and_views:
- query = f"describe formatted `{schema}`.`{table_view}`"
- cursor = connection.execute(query)
- results = cursor.fetchall()
-
- for result in list(results):
- data = result
- if data[0].strip() == "Table Type:":
- if target_type.lower() in data[1].lower():
- retvalue.append(table_view)
- return retvalue
-
-
-def get_impala_view_names(
- self, connection, schema=None, **kw
-): # pylint: disable=unused-argument
- results = get_impala_table_or_view_names(connection, schema, "view")
- return results
-
-
-def get_impala_table_names(
- self, connection, schema=None, **kw
-): # pylint: disable=unused-argument
- results = get_impala_table_or_view_names(connection, schema, "table")
- return results
-
-
-def get_impala_table_comment(
- self, connection, table_name, schema_name, **kw
-): # pylint: disable=unused-argument
- """
- Gets the table comment from the describe formatted query result under the Table Parameters section.
- """
- full_table_name = (
- f"{schema_name}.{table_name}" if schema_name is not None else table_name
- )
- split_name = full_table_name.split(".")
- query = f"describe formatted `{split_name[0]}`.`{split_name[1]}`"
- cursor = connection.execute(query)
- results = cursor.fetchall()
-
- found_table_parameters = False
- try:
- for result in list(results):
- data = result
- if not found_table_parameters and data[0].strip() == "Table Parameters:":
- found_table_parameters = True
- if found_table_parameters:
- coltext = data[1].strip() if data[1] is not None else ""
- if coltext == "comment":
- return {"text": data[2]}
- except Exception:
- return {"text": None}
- return {"text": None}
-
-
-def get_impala_columns(
- self, connection, table_name, schema=None, **kwargs
-): # pylint: disable=unused-argument
- # pylint: disable=too-many-locals
- """
- Extracted from the Impala Dialect. We'll tune the implementation.
-
- By default, this gives us the column name as `table.column`. We just
- want to get `column`.
- """
- full_table_name = f"{schema}.{table_name}" if schema is not None else table_name
- split_name = full_table_name.split(".")
- query = f"DESCRIBE `{split_name[0]}`.`{split_name[1]}`"
- describe_table_rows = connection.execute(query)
- column_info = []
- ordinal_pos = 0
- for col in describe_table_rows:
- ordinal_pos = ordinal_pos + 1
- col_raw = col[1]
- attype = re.sub(r"\(.*\)", "", col[1])
- col_type = re.search(r"^\w+", col[1]).group(0)
- try:
- coltype = _type_map[col_type]
- except KeyError:
- util.warn(f"Did not recognize type '{col_raw}' of column '{col[0]}'")
- coltype = types.NullType
- charlen = re.search(r"\(([\d,]+)\)", col_raw.lower())
- if charlen:
- charlen = charlen.group(1)
- if attype == "decimal":
- prec, scale = charlen.split(",")
- args = (int(prec), int(scale))
- else:
- args = (int(charlen),)
- coltype = coltype(*args)
- add_column = {
- "name": col[0],
- "type": coltype,
- "comment": col[2],
- "nullable": True,
- "autoincrement": False,
- "ordinalPosition": ordinal_pos,
- }
- column_info.append(add_column)
- return column_info
-
-
 # pylint: disable=unused-argument
 @reflection.cache
 def get_view_definition(self, connection, view_name, schema=None, **kw):
@@ -313,25 +193,9 @@ def get_view_definition(self, connection, view_name, schema=None, **kw):
  return None
 
 
-# pylint: disable=unused-argument
-@reflection.cache
-def get_impala_view_definition(self, connection, view_name, schema=None, **kw):
- """
- Gets the view definition
- """
- full_view_name = f"`{view_name}`" if not schema else f"`{schema}`.`{view_name}`"
- res = connection.execute(f"SHOW CREATE VIEW {full_view_name}").fetchall()
- if res:
- return "\n".join(i[0] for i in res)
- return None
-
-
 HiveDialect.get_columns = get_columns
 HiveDialect.get_table_comment = get_table_comment
 
-ImpalaDialect.get_columns = get_impala_columns
-ImpalaDialect.get_table_comment = get_impala_table_comment
-ImpalaDialect.get_view_definition = get_impala_view_definition
 
 HIVE_VERSION_WITH_VIEW_SUPPORT = "2.2.0"
 
@@ -363,22 +227,15 @@ def prepare(self):
  Fetching views in hive server with query "SHOW VIEWS" was possible
  only after hive 2.2.0 version
  """
- if self.engine.driver == Dialects.Impala:
- ImpalaDialect.get_table_names = get_impala_table_names
- ImpalaDialect.get_view_names = get_impala_view_names
- ImpalaDialect.get_table_comment = get_impala_table_comment
- ImpalaDialect.get_columns = get_impala_columns
- ImpalaDialect.get_view_definition = get_impala_view_definition
+ result = dict(self.engine.execute("SELECT VERSION()").fetchone())
+
+ version = result.get("_c0", "").split()
+ if version and self._parse_version(version[0]) >= self._parse_version(
+ HIVE_VERSION_WITH_VIEW_SUPPORT
+ ):
+ HiveDialect.get_table_names = get_table_names
+ HiveDialect.get_view_names = get_view_names
+ HiveDialect.get_view_definition = get_view_definition
  else:
- result = dict(self.engine.execute("SELECT VERSION()").fetchone())
-
- version = result.get("_c0", "").split()
- if version and self._parse_version(version[0]) >= self._parse_version(
- HIVE_VERSION_WITH_VIEW_SUPPORT
- ):
- HiveDialect.get_table_names = get_table_names
- HiveDialect.get_view_names = get_view_names
- HiveDialect.get_view_definition = get_view_definition
- else:
- HiveDialect.get_table_names = get_table_names_older_versions
- HiveDialect.get_view_names = get_view_names_older_versions
+ HiveDialect.get_table_names = get_table_names_older_versions
+ HiveDialect.get_view_names = get_view_names_older_versions

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/hiveConnection.json

@@ -15,7 +15,7 @@
  "hiveScheme": {
  "description": "SQLAlchemy driver scheme options.",
  "type": "string",
- "enum": ["hive", "hive+http", "hive+https", "impala", "impala4"],
+ "enum": ["hive", "hive+http", "hive+https"],
  "default": "hive"
  }
  },

openmetadata-ui/src/main/resources/ui/cypress/common/common.js

@@ -1014,9 +1014,12 @@ export const updateOwner = () => {
  cy.get('[data-testid="hiden-layer"]').should('exist').click();
  interceptURL('GET', '/api/v1/users?limit=15', 'getUsers');
  // Clicking on edit owner button
- cy.get('[data-testid="add-user"]').should('be.visible').click();
+ cy.get('[data-testid="edit-owner"]').should('be.visible').click();
+
+ cy.get('.user-team-select-popover').contains('Users').click();
 
  cy.get('[data-testid="selectable-list"]')
+ .eq(1)
  .find(`[title="${text.trim()}"]`)
  .click();
 

openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/RestoreEntity.spec.js

@@ -117,7 +117,7 @@ describe('Restore entity functionality should work properly', () => {
 
  interceptURL(
  'GET',
- '/api/v1/search/query?q=*&index=table_search_index&from=0&size=10&deleted=true&sort_field=name.keyword&sort_order=asc',
+ '/api/v1/tables?databaseSchema=sample_data.ecommerce_db.shopify&include=deleted',
  'queryDeletedTables'
  );
 

openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddAndRemoveTierAndOwner.spec.js

@@ -205,14 +205,28 @@ describe('Add and Remove Owner and Tier', () => {
  verifyResponseStatusCode('@getOrganization', 200);
  verifyResponseStatusCode('@teamPermission', 200);
 
- cy.get('[data-testid="add-user"]').should('be.visible').click();
- verifyResponseStatusCode('@getUsers', 200);
- cy.get(`.ant-popover [title="${OWNER}"]`).should('be.visible').click();
+ interceptURL(
+ 'GET',
+ 'api/v1/search/query?q=**%20AND%20isBot:false&from=0&size=0&index=user_search_index',
+ 'waitForUsers'
+ );
+
+ // Click on edit owner button
+ cy.get('[data-testid="edit-owner"]').click();
+ verifyResponseStatusCode('@waitForUsers', 200);
+
+ cy.get('.user-team-select-popover').contains('Users').click();
+
+ cy.get('[data-testid="selectable-list"]')
+ .eq(1)
+ .find(`[title="${OWNER}"]`)
+ .click();
+
  verifyResponseStatusCode('@patchOwner', 200);
  cy.get('[data-testid="owner-link"]')
  .should('be.visible')
  .should('contain', OWNER);
- cy.get('[data-testid="add-user"]').should('be.visible').click();
+ cy.get('[data-testid="edit-owner"]').should('be.visible').click();
  verifyResponseStatusCode('@getUsers', 200);
  cy.get('[data-testid="remove-owner"]').should('be.visible').click();
  verifyResponseStatusCode('@patchOwner', 200);

openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js

@@ -105,7 +105,6 @@ describe('Services page should work properly', () => {
 
  verifyResponseStatusCode('@updateService', 200);
 
- cy.get('[data-testid="owner-dropdown"]').should('have.text', service.Owner);
  // Checking if description exists after assigning the owner
  cy.get(':nth-child(1) > .link-title').click();
  // need wait here

openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/CustomControls.component.tsx

@@ -148,7 +148,7 @@ const CustomControls: FC<ControlProps> = ({
  <SVGIcons
  alt="icon-edit-lineag"
  className="m--t-xss"
- icon={isEditMode ? 'icon-edit-lineage-color' : 'icon-edit-lineage'}
+ icon={isEditMode ? 'icon-edit-lineage' : 'icon-edit-lineage-color'}
  width="14"
  />
  );
@@ -270,8 +270,7 @@ const CustomControls: FC<ControlProps> = ({
  className={classNames(
  'custom-control-edit-button h-8 w-8 rounded-full p-x-xss',
  {
- 'bg-primary': !isEditMode,
- 'bg-primary-hover-lite': isEditMode,
+ 'bg-primary': isEditMode,
  }
  )}
  data-testid="edit-lineage"

openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/CustomEdge.component.tsx

@@ -122,10 +122,9 @@ export const CustomEdge = ({
  return (
  <LineageEdgeIcon offset={3} x={edgeCenterX} y={edgeCenterY}>
  <Button
- className="d-flex justify-center items-center custom-edge-pipeline-button"
+ className="flex-center custom-edge-pipeline-button"
  data-testid={dataTestId}
  icon={icon}
- type="primary"
  onClick={(event) =>
  data.isEditMode &&
  data.addPipelineClick?.(event, rest as CustomEdgeData)
@@ -187,7 +186,10 @@ export const CustomEdge = ({
 
  {isColumnLineageAllowed &&
  hasLabel &&
- getLineageEdgeIcon(<PipelineIcon />, 'pipeline-label')}
+ getLineageEdgeIcon(
+ <PipelineIcon className="text-primary" />,
+ 'pipeline-label'
+ )}
  {isColumnLineageAllowed &&
  isSelectedEditMode &&
  getEditLineageIcon('add-pipeline', true, addPipelineClick)}

openmetadata-ui/src/main/resources/ui/src/components/EntityLineage/custom-node.less

@@ -11,6 +11,9 @@
  * limitations under the License.
  */
 @import url('../../styles/variables.less');
+
+@lineage-border: #b1b1b7;
+
 .lineage-node-card {
  .ant-card-body {
  padding: 0 !important;
@@ -40,7 +43,7 @@
 }
 
 .lineage-node {
- border: @global-border;
+ border: 1px solid @lineage-border;
  border-radius: 4px;
 }
 
@@ -69,10 +72,12 @@
 .react-flow__node-input.selectable:hover,
 .react-flow__node-output.selectable:hover,
 .react-flow__node-group.selectable:hover {
- border-color: @primary-color !important;
  .lineage-node-handle {
  border-color: @primary-color;
  }
+ .lineage-node {
+ border-color: @primary-color !important;
+ }
 }
 
 .react-flow__node-default.selectable.selected,
@@ -122,15 +127,15 @@
  width: 20px;
  height: 20px;
  border-radius: 50%;
- border-color: @border-color;
+ border-color: @lineage-border;
  background: @white;
  top: 38px; // Need to show handles on top half
 }
 
 .react-flow .lineage-column-node-handle {
  width: 20px;
  border-radius: 0;
- border-color: @border-color;
+ border-color: @lineage-border;
  background: @white;
  top: 1px;
  height: 25px;