Skip to content

Commit

Permalink
MINOR: Fix column merge error lineage (#16670)
Browse files Browse the repository at this point in the history
* MINOR: Fix Column merge error

* append only when col lineage available

* pyformat

(cherry picked from commit 72ed09d)
  • Loading branch information
ulixius9 authored and akash-jain-10 committed Jun 17, 2024
1 parent dd56711 commit 502e5c9
Showing 1 changed file with 18 additions and 5 deletions.
23 changes: 18 additions & 5 deletions ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,24 @@ def _merge_column_lineage(
self, original: List[Dict[str, Any]], updated: List[Dict[str, Any]]
):
temp_result = []
for column in original or []:
temp_result.append((*column.get("fromColumns", []), column.get("toColumn")))
for column in updated or []:
data = column.dict()
temp_result.append((*data.get("fromColumns", []), data.get("toColumn")))
try:
for column in original or []:
if column.get("toColumn") and column.get("fromColumns"):
temp_result.append(
(*column.get("fromColumns", []), column.get("toColumn"))
)
for column in updated or []:
if not isinstance(column, dict):
data = column.dict()
else:
data = column
if data.get("toColumn") and data.get("fromColumns"):
temp_result.append(
(*data.get("fromColumns", []), data.get("toColumn"))
)
except Exception as exc:
logger.debug(f"Error while merging column lineage: {exc}")
logger.debug(traceback.format_exc())
return [
{"fromColumns": list(col_data[:-1]), "toColumn": col_data[-1]}
for col_data in set(temp_result)
Expand Down

0 comments on commit 502e5c9

Please sign in to comment.