Create Change Events on import and update in es (#18953)

* Create Change Events on import and update in es * add glossary tests * minor test cleanup * Make Change Event Update Async * Fix Test case --------- Co-authored-by: karanh37 <karanh37@gmail.com> Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com>
open-metadata · Dec 9, 2024 · 5263858 · 5263858
1 parent de0889d
commit 5263858
Show file tree

Hide file tree

Showing 8 changed files with 293 additions and 123 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
@@ -23,6 +23,7 @@
 import static org.openmetadata.csv.CsvUtil.fieldToExtensionStrings;
 import static org.openmetadata.csv.CsvUtil.fieldToInternalArray;
 import static org.openmetadata.csv.CsvUtil.recordToString;
+import static org.openmetadata.service.events.ChangeEventHandler.copyChangeEvent;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.networknt.schema.JsonSchema;
@@ -60,7 +61,9 @@
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.type.ApiStatus;
+import org.openmetadata.schema.type.ChangeEvent;
 import org.openmetadata.schema.type.EntityReference;
+import org.openmetadata.schema.type.EventType;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.schema.type.TagLabel.TagSource;
@@ -72,7 +75,9 @@
 import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.TypeRegistry;
+import org.openmetadata.service.formatter.util.FormatterUtil;
 import org.openmetadata.service.jdbi3.EntityRepository;
+import org.openmetadata.service.util.AsyncService;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.RestUtil.PutResponse;
@@ -721,6 +726,9 @@ protected void createEntity(CSVPrinter resultsPrinter, CSVRecord csvRecord, T en
         repository.prepareInternal(entity, false);
         PutResponse<T> response = repository.createOrUpdate(null, entity);
         responseStatus = response.getStatus();
+        AsyncService.getInstance()
+            .getExecutorService()
+            .submit(() -> createChangeEventAndUpdateInES(response, importedBy));
       } catch (Exception ex) {
         importFailure(resultsPrinter, ex.getMessage(), csvRecord);
         importResult.setStatus(ApiStatus.FAILURE);
@@ -744,6 +752,20 @@ protected void createEntity(CSVPrinter resultsPrinter, CSVRecord csvRecord, T en
     }
   }
 
+  private void createChangeEventAndUpdateInES(PutResponse<T> response, String importedBy) {
+    if (!response.getChangeType().equals(EventType.ENTITY_NO_CHANGE)) {
+      ChangeEvent changeEvent =
+          FormatterUtil.createChangeEventForEntity(
+              importedBy, response.getChangeType(), response.getEntity());
+      Object entity = changeEvent.getEntity();
+      changeEvent = copyChangeEvent(changeEvent);
+      changeEvent.setEntity(JsonUtils.pojoToMaskedJson(entity));
+      // Change Event and Update in Es
+      Entity.getCollectionDAO().changeEventDAO().insert(JsonUtils.pojoToJson(changeEvent));
+      Entity.getSearchRepository().updateEntity(response.getEntity().getEntityReference());
+    }
+  }
+
   @Transaction
   protected void createUserEntity(CSVPrinter resultsPrinter, CSVRecord csvRecord, T entity)
       throws IOException {

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java
@@ -86,7 +86,7 @@ public Void process(
     return null;
   }
 
-  private static ChangeEvent copyChangeEvent(ChangeEvent changeEvent) {
+  public static ChangeEvent copyChangeEvent(ChangeEvent changeEvent) {
     return new ChangeEvent()
         .withId(changeEvent.getId())
         .withEventType(changeEvent.getEventType())

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java b/...metadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java
@@ -253,7 +253,7 @@ private static ChangeEvent extractChangeEvent(
     return null;
   }
 
-  private static ChangeEvent createChangeEventForEntity(
+  public static ChangeEvent createChangeEventForEntity(
       String updateBy, EventType eventType, EntityInterface entityInterface) {
     return getChangeEvent(
             updateBy, eventType, entityInterface.getEntityReference().getType(), entityInterface)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java
@@ -122,7 +122,7 @@ public static class PutResponse<T> {
     @Getter private T entity;
     private ChangeEvent changeEvent;
     @Getter private final Response.Status status;
-    private final EventType changeType;
+    @Getter private final EventType changeType;
 
     /**
      * Response.Status.CREATED when PUT operation creates a new entity or Response.Status.OK when PUT operation updates

diff --git a/...rvice/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/...rvice/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
@@ -934,8 +934,25 @@ void testGlossaryImportExport() throws IOException {
     List<String> newRecords =
         listOf(
             ",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"");
-    testImportExport(
-        glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords);
+    Awaitility.await()
+        .atMost(Duration.ofMillis(120 * 1000L))
+        .pollInterval(Duration.ofMillis(2000L))
+        .ignoreExceptions()
+        .until(
+            () -> {
+              try {
+                testImportExport(
+                    glossary.getName(),
+                    GlossaryCsv.HEADERS,
+                    createRecords,
+                    updateRecords,
+                    newRecords);
+                return true;
+              } catch (Exception e) {
+                // Return false to retry
+                return false;
+              }
+            });
   }
 
   @Test

diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts
@@ -50,11 +50,11 @@ import {
   createGlossaryTerms,
   createTagTaskForGlossary,
   deleteGlossaryOrGlossaryTerm,
-  descriptionBox,
   deselectColumns,
   dragAndDropColumn,
   dragAndDropTerm,
   filterStatus,
+  getEscapedTermFqn,
   goToAssetsTab,
   openColumnDropdown,
   renameGlossaryTerm,
@@ -63,6 +63,8 @@ import {
   selectColumns,
   toggleAllColumnsSelection,
   updateGlossaryTermDataFromTree,
+  updateGlossaryTermOwners,
+  updateGlossaryTermReviewers,
   validateGlossaryTerm,
   verifyAllColumns,
   verifyColumnsVisibility,
@@ -126,7 +128,7 @@ test.describe('Glossary tests', () => {
         await verifyTaskCreated(
           page1,
           glossary1.data.fullyQualifiedName,
-          glossary1.data.terms[0].data
+          glossary1.data.terms[0].data.name
         );
 
         await approveGlossaryTermTask(page1, glossary1.data.terms[0].data);
@@ -183,7 +185,7 @@ test.describe('Glossary tests', () => {
         await verifyTaskCreated(
           page1,
           glossary2.data.fullyQualifiedName,
-          glossary2.data.terms[0].data
+          glossary2.data.terms[0].data.name
         );
 
         await approveGlossaryTermTask(page1, glossary2.data.terms[0].data);
@@ -304,126 +306,63 @@ test.describe('Glossary tests', () => {
     const { page, afterAction, apiContext } = await performAdminLogin(browser);
     const glossary1 = new Glossary();
     const glossaryTerm1 = new GlossaryTerm(glossary1);
-    await glossary1.create(apiContext);
-    await glossaryTerm1.create(apiContext);
-
-    const testtag = `Test tag ${uuid()}`;
-
-    await redirectToHomePage(page);
-    await sidebarClick(page, SidebarItem.GLOSSARY);
-
-    await page.click('[data-testid="add-classification"]');
-
-    await expect(page.getByRole('dialog')).toBeVisible();
-
-    await page.getByTestId('name').fill(testtag);
-    await page.locator(descriptionBox).fill('Test Description');
+    const owner1 = new UserClass();
+    const reviewer1 = new UserClass();
 
-    const glossaryTermResponse = page.waitForResponse('/api/v1/glossaryTerms');
-    await page.click('[data-testid="save-glossary-term"]');
-    await glossaryTermResponse;
-
-    await page.click('[data-testid="expand-icon"]');
-
-    await expect(page.getByText(testtag)).toBeVisible();
-
-    await page.getByTestId('edit-button').last().click();
+    try {
+      await glossary1.create(apiContext);
+      await glossaryTerm1.create(apiContext);
+      await owner1.create(apiContext);
+      await reviewer1.create(apiContext);
+      await await redirectToHomePage(page);
+      await sidebarClick(page, SidebarItem.GLOSSARY);
+      await selectActiveGlossary(page, glossary1.data.displayName);
 
-    await expect(page.getByRole('dialog')).toBeVisible();
+      await updateGlossaryTermOwners(page, glossaryTerm1.data, [
+        {
+          name: `${owner1.data.firstName}${owner1.data.lastName}`,
+          type: 'user',
+        },
+      ]);
 
-    await page
-      .getByTestId('edit-glossary-modal')
-      .getByTestId('add-owner')
-      .click();
+      await updateGlossaryTermReviewers(page, glossaryTerm1.data, [
+        {
+          name: `${reviewer1.data.firstName}${reviewer1.data.lastName}`,
+          type: 'user',
+        },
+      ]);
 
-    await expect(page.getByTestId('select-owner-tabs')).toBeVisible();
+      await openColumnDropdown(page);
+      const checkboxLabels = ['Reviewer'];
+      await selectColumns(page, checkboxLabels);
+      await clickSaveButton(page);
+      await verifyColumnsVisibility(page, checkboxLabels, true);
 
-    await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
+      const escapedFqn = getEscapedTermFqn(glossaryTerm1.data);
+      const termRow = page.locator(`[data-row-key="${escapedFqn}"]`);
 
-    await expect(page.getByRole('listitem', { name: 'admin' })).toHaveClass(
-      /active/
-    );
+      // Verify the Reviewer
+      const reviewerSelector = `td:nth-child(3) a[data-testid="owner-link"]`;
+      const reviewerText = await termRow
+        .locator(reviewerSelector)
+        .textContent();
 
-    const searchUser = page.waitForResponse(
-      `/api/v1/search/query?q=*${encodeURIComponent(
-        user1.responseData.displayName
-      )}*`
-    );
-    await page
-      .getByTestId(`owner-select-users-search-bar`)
-      .fill(user1.responseData.displayName);
-    await searchUser;
-
-    await page
-      .getByRole('listitem', {
-        name: user1.responseData.displayName,
-        exact: true,
-      })
-      .click();
-    await page.getByTestId('selectable-list-update-btn').click();
-
-    await expect(
-      page.getByRole('link', { name: user1.responseData.displayName })
-    ).toBeVisible();
-
-    await page
-      .getByTestId('edit-glossary-modal')
-      .getByTestId('add-reviewers')
-      .click();
-
-    await expect(
-      page
-        .getByRole('tooltip', { name: 'Selected Reviewers Teams' })
-        .getByTestId('select-owner-tabs')
-    ).toBeVisible();
-
-    const userListResponse = page.waitForResponse(
-      '/api/v1/users?limit=*&isBot=false*'
-    );
-    await page.getByRole('tab', { name: 'Users' }).click();
-    await userListResponse;
+      expect(reviewerText).toBe(
+        `${reviewer1.data.firstName}${reviewer1.data.lastName}`
+      );
 
-    await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
+      // Verify the Owner
+      const ownerSelector = `td:nth-child(4) a[data-testid="owner-link"]`;
+      const ownerText = await termRow.locator(ownerSelector).textContent();
 
-    const searchUserReviewer = page.waitForResponse(
-      `/api/v1/search/query?q=*${encodeURIComponent(
-        user2.responseData.displayName
-      )}*`
-    );
-    await page
-      .getByTestId(`owner-select-users-search-bar`)
-      .fill(user2.responseData.displayName);
-    await searchUserReviewer;
-
-    await page
-      .getByRole('listitem', {
-        name: user2.responseData.displayName,
-        exact: true,
-      })
-      .click();
-
-    await page.getByTestId('selectable-list-update-btn').click();
-
-    await page.click('[data-testid="save-glossary-term"]');
-    await glossaryTermResponse;
-
-    await expect(
-      page.getByRole('link', { name: user1.responseData.displayName })
-    ).toBeVisible();
-
-    await openColumnDropdown(page);
-    const checkboxLabels = ['Reviewer'];
-    await selectColumns(page, checkboxLabels);
-    await clickSaveButton(page);
-    await verifyColumnsVisibility(page, checkboxLabels, true);
-
-    await expect(
-      page.getByRole('link', { name: user2.responseData.displayName })
-    ).toBeVisible();
-
-    await glossaryTerm1.delete(apiContext);
-    await glossary1.delete(apiContext);
-    await afterAction();
+      expect(ownerText).toBe(`${owner1.data.firstName}${owner1.data.lastName}`);
+    } finally {
+      await glossaryTerm1.delete(apiContext);
+      await glossary1.delete(apiContext);
+      await owner1.delete(apiContext);
+      await reviewer1.delete(apiContext);
+      await afterAction();
+    }
   });
 
   test('Add and Remove Assets', async ({ browser }) => {