From 502e5c9e46ecb6f1e9f757e5ffc25bb84c3b0509 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Mon, 17 Jun 2024 09:02:21 +0530 Subject: [PATCH] MINOR: Fix column merge error lineage (#16670) * MINOR: Fix Column merge error * append only when col lineage available * pyformat (cherry picked from commit 72ed09d6f63c7d00227cb7407c15e65a579445fe) --- .../ingestion/ometa/mixins/lineage_mixin.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py index 543b12c5459a..601b62e628fe 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py @@ -52,11 +52,24 @@ def _merge_column_lineage( self, original: List[Dict[str, Any]], updated: List[Dict[str, Any]] ): temp_result = [] - for column in original or []: - temp_result.append((*column.get("fromColumns", []), column.get("toColumn"))) - for column in updated or []: - data = column.dict() - temp_result.append((*data.get("fromColumns", []), data.get("toColumn"))) + try: + for column in original or []: + if column.get("toColumn") and column.get("fromColumns"): + temp_result.append( + (*column.get("fromColumns", []), column.get("toColumn")) + ) + for column in updated or []: + if not isinstance(column, dict): + data = column.dict() + else: + data = column + if data.get("toColumn") and data.get("fromColumns"): + temp_result.append( + (*data.get("fromColumns", []), data.get("toColumn")) + ) + except Exception as exc: + logger.debug(f"Error while merging column lineage: {exc}") + logger.debug(traceback.format_exc()) return [ {"fromColumns": list(col_data[:-1]), "toColumn": col_data[-1]} for col_data in set(temp_result)