-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix test suite migrations to 1.1.4 (#13042)
* Fix test suite migrations to 1.1.4 * Fix test suite migrations to 1.1.4 * Fix test suite migrations to 1.1.4 * Fix test suite migrations to 1.1.4
- Loading branch information
Showing
8 changed files
with
356 additions
and
193 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
...tadata-service/src/main/java/org/openmetadata/service/migration/mysql/v114/Migration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package org.openmetadata.service.migration.mysql.v114; | ||
|
|
||
| import static org.openmetadata.service.migration.utils.V112.MigrationUtil.lowerCaseUserNameAndEmail; | ||
| import static org.openmetadata.service.migration.utils.V114.MigrationUtil.fixTestSuites; | ||
|
|
||
| import lombok.SneakyThrows; | ||
| import org.jdbi.v3.core.Handle; | ||
| import org.openmetadata.service.jdbi3.CollectionDAO; | ||
| import org.openmetadata.service.migration.api.MigrationProcessImpl; | ||
| import org.openmetadata.service.migration.utils.MigrationFile; | ||
|
|
||
| public class Migration extends MigrationProcessImpl { | ||
| private CollectionDAO collectionDAO; | ||
|
|
||
| public Migration(MigrationFile migrationFile) { | ||
| super(migrationFile); | ||
| } | ||
|
|
||
| @Override | ||
| public void initialize(Handle handle) { | ||
| super.initialize(handle); | ||
| this.collectionDAO = handle.attach(CollectionDAO.class); | ||
| } | ||
|
|
||
| @Override | ||
| @SneakyThrows | ||
| public void runDataMigration() { | ||
| fixTestSuites(collectionDAO); | ||
| lowerCaseUserNameAndEmail(collectionDAO); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
...ata-service/src/main/java/org/openmetadata/service/migration/postgres/v114/Migration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package org.openmetadata.service.migration.postgres.v114; | ||
|
|
||
| import static org.openmetadata.service.migration.utils.V112.MigrationUtil.lowerCaseUserNameAndEmail; | ||
| import static org.openmetadata.service.migration.utils.V114.MigrationUtil.fixTestSuites; | ||
|
|
||
| import lombok.SneakyThrows; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.jdbi.v3.core.Handle; | ||
| import org.openmetadata.service.jdbi3.CollectionDAO; | ||
| import org.openmetadata.service.migration.api.MigrationProcessImpl; | ||
| import org.openmetadata.service.migration.utils.MigrationFile; | ||
|
|
||
| @Slf4j | ||
| public class Migration extends MigrationProcessImpl { | ||
| private CollectionDAO collectionDAO; | ||
| private Handle handle; | ||
|
|
||
| public Migration(MigrationFile migrationFile) { | ||
| super(migrationFile); | ||
| } | ||
|
|
||
| @Override | ||
| public void initialize(Handle handle) { | ||
| super.initialize(handle); | ||
| this.handle = handle; | ||
| this.collectionDAO = handle.attach(CollectionDAO.class); | ||
| } | ||
|
|
||
| @Override | ||
| @SneakyThrows | ||
| public void runDataMigration() { | ||
| fixTestSuites(collectionDAO); | ||
| lowerCaseUserNameAndEmail(collectionDAO); | ||
| } | ||
| } |
52 changes: 52 additions & 0 deletions
52
...ta-service/src/main/java/org/openmetadata/service/migration/utils/V112/MigrationUtil.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package org.openmetadata.service.migration.utils.V112; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Set; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.openmetadata.schema.entity.teams.User; | ||
| import org.openmetadata.schema.tests.TestSuite; | ||
| import org.openmetadata.schema.type.Include; | ||
| import org.openmetadata.schema.utils.EntityInterfaceUtil; | ||
| import org.openmetadata.service.jdbi3.CollectionDAO; | ||
| import org.openmetadata.service.jdbi3.ListFilter; | ||
| import org.openmetadata.service.jdbi3.TestSuiteRepository; | ||
| import org.openmetadata.service.util.EntityUtil; | ||
| import org.openmetadata.service.util.JsonUtils; | ||
|
|
||
| @Slf4j | ||
| public class MigrationUtil { | ||
| private MigrationUtil() {} | ||
|
|
||
| public static void fixExecutableTestSuiteFQN(CollectionDAO collectionDAO) { | ||
| TestSuiteRepository testSuiteRepository = new TestSuiteRepository(collectionDAO); | ||
| List<TestSuite> testSuites = | ||
| testSuiteRepository.listAll(new EntityUtil.Fields(Set.of("id")), new ListFilter(Include.ALL)); | ||
| for (TestSuite suite : testSuites) { | ||
| if (Boolean.TRUE.equals(suite.getExecutable()) && suite.getExecutableEntityReference() != null) { | ||
| String tableFQN = suite.getExecutableEntityReference().getFullyQualifiedName(); | ||
| String suiteFQN = tableFQN + ".testSuite"; | ||
| suite.setName(suiteFQN); | ||
| suite.setFullyQualifiedName(suiteFQN); | ||
| collectionDAO.testSuiteDAO().update(suite); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public static void lowerCaseUserNameAndEmail(CollectionDAO daoCollection) { | ||
| LOG.debug("Starting Migration UserName and Email to Lowercase"); | ||
| int total = daoCollection.userDAO().listTotalCount(); | ||
| int offset = 0; | ||
| int limit = 200; | ||
| while (offset < total) { | ||
| List<String> userEntities = daoCollection.userDAO().listAfterWithOffset(limit, offset); | ||
| for (String json : userEntities) { | ||
| User userEntity = JsonUtils.readValue(json, User.class); | ||
| userEntity.setFullyQualifiedName( | ||
| EntityInterfaceUtil.quoteName(userEntity.getFullyQualifiedName().toLowerCase())); | ||
| daoCollection.userDAO().update(userEntity); | ||
| } | ||
| offset = offset + limit; | ||
| } | ||
| LOG.debug("Completed Migrating UserName and Email to Lowercase"); | ||
| } | ||
| } |
91 changes: 91 additions & 0 deletions
91
...ta-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| package org.openmetadata.service.migration.utils.V114; | ||
|
|
||
| import static org.openmetadata.service.Entity.TEST_CASE; | ||
| import static org.openmetadata.service.Entity.TEST_SUITE; | ||
| import static org.openmetadata.service.migration.utils.v110.MigrationUtil.groupTestCasesByTable; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
| import org.openmetadata.schema.tests.TestCase; | ||
| import org.openmetadata.schema.tests.TestSuite; | ||
| import org.openmetadata.schema.type.Include; | ||
| import org.openmetadata.schema.type.Relationship; | ||
| import org.openmetadata.service.exception.EntityNotFoundException; | ||
| import org.openmetadata.service.jdbi3.CollectionDAO; | ||
| import org.openmetadata.service.jdbi3.ListFilter; | ||
| import org.openmetadata.service.jdbi3.TestSuiteRepository; | ||
| import org.openmetadata.service.util.EntityUtil; | ||
|
|
||
| public class MigrationUtil { | ||
| private MigrationUtil() { | ||
| /* Cannot create object util class*/ | ||
| } | ||
|
|
||
| /** | ||
| * Step 1: re-run the fix for FQN to catch any issues from previous release where we were quoting the FQN Step 2: | ||
| * Group all the testCases with the table. We will create a Map with Table FQN as the key and all the test cases | ||
| * belonging to that Table Step 3: Iterate through the Map keySet, which is table names. For each table name we create | ||
| * a executable test suite FQN Step 4: Fetch executable testSuite using step 3 FQN Step 5: Iterate through the test | ||
| * case list associated with the current table FQN in the loop Step 6: for each test case fetch TestSuite | ||
| * relationships Step 7: Iterate through the testSuite relation to check if the executableTestSuite FQN matches. If it | ||
| * matches there exists a relation from testCase to a executable Test suite Step 8: If we can't find a match, create a | ||
| * relationship. | ||
| * | ||
| * @param collectionDAO | ||
| */ | ||
| public static void fixTestSuites(CollectionDAO collectionDAO) { | ||
| // Fix any FQN issues for executable TestSuite | ||
| TestSuiteRepository testSuiteRepository = new TestSuiteRepository(collectionDAO); | ||
| List<TestSuite> testSuites = | ||
| testSuiteRepository.listAll(new EntityUtil.Fields(Set.of("id")), new ListFilter(Include.ALL)); | ||
| for (TestSuite suite : testSuites) { | ||
| if (suite.getExecutableEntityReference() != null) { | ||
| String tableFQN = suite.getExecutableEntityReference().getFullyQualifiedName(); | ||
| String suiteFQN = tableFQN + ".testSuite"; | ||
| suite.setName(suiteFQN); | ||
| suite.setFullyQualifiedName(suiteFQN); | ||
| suite.setExecutable(true); | ||
| collectionDAO.testSuiteDAO().update(suite); | ||
| } | ||
| } | ||
| // Let's iterate through the test cases and make sure there exists a relationship between testcases and its native | ||
| // TestSuite | ||
| Map<String, ArrayList<TestCase>> testCasesGroupByTable = groupTestCasesByTable(collectionDAO); | ||
| for (String tableFQN : testCasesGroupByTable.keySet()) { | ||
| List<TestCase> testCases = testCasesGroupByTable.get(tableFQN); | ||
| String executableTestSuiteFQN = tableFQN + ".testSuite"; | ||
| TestSuite executableTestSuite = | ||
| testSuiteRepository.getDao().findEntityByName(executableTestSuiteFQN, "fqnHash", Include.ALL); | ||
| for (TestCase testCase : testCases) { | ||
| // we are setting mustHaveRelationship to "false" to not throw any error. | ||
| List<CollectionDAO.EntityRelationshipRecord> existingRelations = | ||
| testSuiteRepository.findFromRecords(testCase.getId(), TEST_CASE, Relationship.CONTAINS, TEST_SUITE); | ||
| boolean relationWithExecutableTestSuiteExists = false; | ||
| if (existingRelations != null) { | ||
| for (CollectionDAO.EntityRelationshipRecord existingTestSuiteRel : existingRelations) { | ||
| try { | ||
| TestSuite existingTestSuite = testSuiteRepository.getDao().findEntityById(existingTestSuiteRel.getId()); | ||
| if (existingTestSuite.getExecutable() | ||
| && existingTestSuite.getFullyQualifiedName().equals(executableTestSuiteFQN)) { | ||
| // remove the existing relation | ||
| relationWithExecutableTestSuiteExists = true; | ||
| } | ||
| } catch (EntityNotFoundException ex) { | ||
| // if testsuite cannot be retrieved but the relation exists, then this is orphaned realtion, we will | ||
| // delete the relation | ||
| testSuiteRepository.deleteRelationship( | ||
| existingTestSuiteRel.getId(), TEST_SUITE, testCase.getId(), TEST_CASE, Relationship.CONTAINS); | ||
| } | ||
| } | ||
| } | ||
| // if we can't find any executable testSuite relationship add one | ||
| if (!relationWithExecutableTestSuiteExists) { | ||
| testSuiteRepository.addRelationship( | ||
| executableTestSuite.getId(), testCase.getId(), TEST_SUITE, TEST_CASE, Relationship.CONTAINS); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.