From 4ed1a7184e4e9701d72d2df0bc0777d22a0ff441 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Thu, 31 Oct 2024 10:43:47 +0100 Subject: [PATCH 01/11] feat(dataplex): create mvn project for dataplex quickstart --- dataplex/quickstart/pom.xml | 58 +++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 dataplex/quickstart/pom.xml diff --git a/dataplex/quickstart/pom.xml b/dataplex/quickstart/pom.xml new file mode 100644 index 00000000000..07173434647 --- /dev/null +++ b/dataplex/quickstart/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + dataplex + dataplex-quickstart + jar + Google Dataplex Quickstart + + + + com.google.cloud.samples + shared-configuration + 1.2.2 + + + + 11 + 11 + UTF-8 + + + + + + com.google.cloud + libraries-bom + 26.49.0 + pom + import + + + + + + + com.google.cloud + google-cloud-dataplex + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.4.4 + test + + + From 90730ffc0adda5f129168dd7db8804bfc40ee9e1 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Thu, 31 Oct 2024 10:44:09 +0100 Subject: [PATCH 02/11] feat(dataplex): create quickstart guide for dataplex --- .../src/main/java/dataplex/Quickstart.java | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 dataplex/quickstart/src/main/java/dataplex/Quickstart.java diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java new file mode 100644 index 00000000000..69e838bdee9 --- /dev/null +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -0,0 +1,216 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dataplex; + +// [START dataplex_quickstart] +import com.google.cloud.dataplex.v1.Aspect; +import com.google.cloud.dataplex.v1.AspectType; +import com.google.cloud.dataplex.v1.CatalogServiceClient; +import com.google.cloud.dataplex.v1.Entry; +import com.google.cloud.dataplex.v1.EntryGroup; +import com.google.cloud.dataplex.v1.EntryGroupName; +import com.google.cloud.dataplex.v1.EntryName; +import com.google.cloud.dataplex.v1.EntrySource; +import com.google.cloud.dataplex.v1.EntryType; +import com.google.cloud.dataplex.v1.EntryView; +import com.google.cloud.dataplex.v1.GetEntryRequest; +import com.google.cloud.dataplex.v1.LocationName; +import com.google.protobuf.Struct; +import com.google.protobuf.Value; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +public class Quickstart { + + public static void main(String[] args) { + // TODO(developer): Replace these variables before running the sample. + String projectId = "MY_PROJECT_ID"; + // Available locations: https://cloud.google.com/dataplex/docs/locations + String location = "MY_LOCATION"; + + quickstart(projectId, location); + } + + public static void quickstart(String projectId, String location) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + try (CatalogServiceClient client = CatalogServiceClient.create()) { + // 0) Prepare variables used in following steps + String aspectTypeId = "dataplex-quickstart-aspect-type"; + String aspectKey = String.format("%s.global.%s", projectId, aspectTypeId); + String entryTypeId = "dataplex-quickstart-entry-type"; + String entryGroupId = "dataplex-quickstart-entry-group"; + String entryId = "dataplex-quickstart-entry"; + LocationName globalLocationName = LocationName.of(projectId, "global"); + LocationName specificLocationName = LocationName.of(projectId, location); + + // 1) Create Aspect Type that will be attached to Entry Type + AspectType.MetadataTemplate aspectField = + AspectType.MetadataTemplate.newBuilder() + // The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$ + // That means name must only contain alphanumeric character or dashes or underscores, + // start with an alphabet, and must be less than 63 characters. + .setName("example_field") + // Metadata Template is recursive structure, + // primitive types such as "string" or "integer" indicate leaf node, + // complex types such as "record" or "array" would require nested Metadata Template + .setType("string") + .setIndex(1) + .setAnnotations( + AspectType.MetadataTemplate.Annotations.newBuilder() + .setDescription("example field to be filled during entry creation") + .build()) + .setConstraints( + AspectType.MetadataTemplate.Constraints.newBuilder() + // Specifies if field will be required in Aspect Type. + .setRequired(true) + .build()) + .build(); + AspectType aspectType = + AspectType.newBuilder() + .setDescription("aspect type for dataplex quickstart") + .setMetadataTemplate( + AspectType.MetadataTemplate.newBuilder() + .setName("example_template") + .setType("record") + // Aspect Type fields, that themselves are Metadata Templates + .addAllRecordFields(List.of(aspectField)) + .build()) + .build(); + AspectType createdAspectType = + client + .createAspectTypeAsync( + // Aspect Type is created in "global" location to highlight, that resources from + // "global" region can be attached Entry created in specific location + globalLocationName, aspectType, aspectTypeId) + .get(); + System.out.println("Step 1: Created aspect type -> " + createdAspectType.getName()); + + // 2) Create Entry Type, of which type Entry will be created + EntryType entryType = + EntryType.newBuilder() + .setDescription("entry type for dataplex quickstart") + .addRequiredAspects( + EntryType.AspectInfo.newBuilder() + // Aspect Type created in step 1 + .setType( + String.format( + "projects/%s/locations/global/aspectTypes/%s", + projectId, aspectTypeId)) + .build()) + .build(); + EntryType createdEntryType = + client + // Entry Type is created in "global" location to highlight, that resources from + // "global" region can be attached Entry created in specific location + .createEntryTypeAsync(globalLocationName, entryType, entryTypeId) + .get(); + System.out.println("Step 2: Created entry type -> " + createdEntryType.getName()); + + // 3) Create Entry Group in which Entry will be located + EntryGroup entryGroup = + EntryGroup.newBuilder().setDescription("entry group for dataplex quickstart").build(); + EntryGroup createdEntryGroup = + client + // Entry Group is created for specific location + .createEntryGroupAsync(specificLocationName, entryGroup, entryGroupId) + .get(); + System.out.println("Step 3: Created entry group -> " + createdEntryGroup.getName()); + + // 4) Create Entry + Entry entry = + Entry.newBuilder() + .setEntryType( + // Entry is an instance of Entry Type created in step 2 + String.format( + "projects/%s/locations/global/entryTypes/%s", projectId, entryTypeId)) + .setEntrySource( + EntrySource.newBuilder().setDescription("entry for dataplex quickstart").build()) + .putAllAspects( + Map.of( + // Attach Aspect that is an instance of Aspect Type created in step 1 + aspectKey, + Aspect.newBuilder() + .setAspectType( + String.format( + "projects/%s/locations/global/aspectTypes/%s", + projectId, aspectTypeId)) + .setData( + Struct.newBuilder() + .putFields( + "example_field", + Value.newBuilder() + .setStringValue("example value for the field") + .build()) + .build()) + .build())) + .build(); + Entry createdEntry = + client.createEntry( + // Entry is created in specific location, but it is still possible to link it with + // resources (Aspect Type and Entry Type) from "global" location + EntryGroupName.of(projectId, location, entryGroupId), entry, entryId); + System.out.println("Step 4: Created entry -> " + createdEntry.getName()); + + // 5) Retrieve created Entry + GetEntryRequest getEntryRequest = + GetEntryRequest.newBuilder() + .setName(EntryName.of(projectId, location, entryGroupId, entryId).toString()) + .setView(EntryView.FULL) + .build(); + Entry retreivedEntry = client.getEntry(getEntryRequest); + System.out.println("Step 5: Retrieved entry -> " + retreivedEntry.getName()); + retreivedEntry + .getAspectsMap() + .values() + .forEach( + retreivedAspect -> { + System.out.println("Retrieved aspect for entry:"); + System.out.println(" * aspect type -> " + retreivedAspect.getAspectType()); + System.out.println( + " * aspect field value -> " + + retreivedAspect + .getData() + .getFieldsMap() + .get("example_field") + .getStringValue()); + }); + + // 6) Clean created resources + client + .deleteEntryGroupAsync( + String.format( + "projects/%s/locations/%s/entryGroups/%s", projectId, location, entryGroupId)) + .get(); + client + .deleteEntryTypeAsync( + String.format("projects/%s/locations/global/entryTypes/%s", projectId, entryTypeId)) + .get(); + client + .deleteAspectTypeAsync( + String.format("projects/%s/locations/global/aspectTypes/%s", projectId, aspectTypeId)) + .get(); + System.out.println("Step 6: Successfully cleaned up resources"); + + } catch (IOException | InterruptedException | ExecutionException e) { + System.err.println("Error during quickstart execution: " + e); + } + } +} +// [END dataplex_quickstart] From a916da9d0ce07fd1c964d10a22e67f88f411cef0 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Thu, 31 Oct 2024 10:44:38 +0100 Subject: [PATCH 03/11] feat(dataplex): add integration test for quickstart --- .../src/test/java/dataplex/QuickstartIT.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java diff --git a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java new file mode 100644 index 00000000000..1a59ec01c96 --- /dev/null +++ b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java @@ -0,0 +1,95 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dataplex; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class QuickstartIT { + private static final String LOCATION = "us-central1"; + private static final String PROJECT_ID = requireProjectIdEnvVar(); + private static ByteArrayOutputStream bout; + private static PrintStream originalPrintStream; + + private static String requireProjectIdEnvVar() { + String value = System.getenv("GOOGLE_CLOUD_PROJECT"); + assertNotNull( + "Environment variable GOOGLE_CLOUD_PROJECT is required to perform these tests.", value); + return value; + } + + @BeforeClass + public static void checkRequirements() { + requireProjectIdEnvVar(); + } + + @BeforeClass + // Set-up code that will be executed before all tests + public static void setUp() { + // Re-direct print stream to capture logging + bout = new ByteArrayOutputStream(); + originalPrintStream = System.out; + System.setOut(new PrintStream(bout)); + } + + @Test + public void testQuickstart() { + List expectedLogs = + List.of( + String.format( + "Step 1: Created aspect type -> projects/%s/locations/global/aspectTypes" + + "/dataplex-quickstart-aspect-type", + PROJECT_ID), + String.format( + "Step 2: Created entry type -> projects/%s/locations/global/entryTypes" + + "/dataplex-quickstart-entry-type", + PROJECT_ID), + String.format( + "Step 3: Created entry group -> projects/%s/locations/%s/entryGroups" + + "/dataplex-quickstart-entry-group", + PROJECT_ID, LOCATION), + String.format( + "Step 4: Created entry -> projects/%s/locations/%s/entryGroups" + + "/dataplex-quickstart-entry-group/entries/dataplex-quickstart-entry", + PROJECT_ID, LOCATION), + String.format( + "Step 5: Retrieved entry -> projects/%s/locations/%s/entryGroups" + + "/dataplex-quickstart-entry-group/entries/dataplex-quickstart-entry", + PROJECT_ID, LOCATION), + "Step 6: Successfully cleaned up resources"); + + Quickstart.quickstart(PROJECT_ID, LOCATION); + String output = bout.toString(); + + expectedLogs.forEach(expectedLog -> assertThat(output).contains(expectedLog)); + } + + @AfterClass + // Clean-up code that will be executed after all tests + public static void tearDown() { + // Restore print statements + System.setOut(originalPrintStream); + bout.reset(); + } +} From b6fd0c3494ae616257bfa4df342f5399090e0b80 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Mon, 4 Nov 2024 09:45:07 +0100 Subject: [PATCH 04/11] feat(dataplex): add method comment for quickstart --- dataplex/quickstart/src/main/java/dataplex/Quickstart.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java index 69e838bdee9..0ff74a07b52 100644 --- a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -47,6 +47,9 @@ public static void main(String[] args) { quickstart(projectId, location); } + // Method to demonstrate lifecycle of different Dataplex resources and their interactions. + // Method creates Aspect Type, Entry Type, Entry Group and Entry, retrieves Entry + // and cleans up created resources. public static void quickstart(String projectId, String location) { // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. From 0a974aa7340fcf1f721a8fb5999aae2d919b6d8e Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Tue, 5 Nov 2024 15:23:33 +0100 Subject: [PATCH 05/11] feat(dataplex): add Search to quickstart --- .../src/main/java/dataplex/Quickstart.java | 27 +++++++++++++++++-- .../src/test/java/dataplex/QuickstartIT.java | 5 +++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java index 0ff74a07b52..0300bb80d52 100644 --- a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -29,12 +29,15 @@ import com.google.cloud.dataplex.v1.EntryView; import com.google.cloud.dataplex.v1.GetEntryRequest; import com.google.cloud.dataplex.v1.LocationName; +import com.google.cloud.dataplex.v1.SearchEntriesRequest; +import com.google.cloud.dataplex.v1.SearchEntriesResult; import com.google.protobuf.Struct; import com.google.protobuf.Value; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; public class Quickstart { @@ -195,7 +198,27 @@ public static void quickstart(String projectId, String location) { .getStringValue()); }); - // 6) Clean created resources + // 6) Use Search capabilities to find Entry + // Wait 30 second to allow resources to propagate to Search + System.out.println("Step 6: Waiting for resources to propagate to Search..."); + Thread.sleep(30000); + SearchEntriesRequest searchEntriesRequest = + SearchEntriesRequest.newBuilder() + .setName(globalLocationName.toString()) + .setQuery("name:dataplex-quickstart-entry") + .build(); + CatalogServiceClient.SearchEntriesPagedResponse searchEntriesResponse = + client.searchEntries(searchEntriesRequest); + List entriesFromSearch = + searchEntriesResponse.getPage().getResponse().getResultsList().stream() + .map(SearchEntriesResult::getDataplexEntry) + .collect(Collectors.toList()); + System.out.println("Entries found in Search:"); + // Please note in log outputs that Entry Group and Entry Type are also represented as Entries + entriesFromSearch.forEach( + entryFromSearch -> System.out.println(" * " + entryFromSearch.getName())); + + // 7) Clean created resources client .deleteEntryGroupAsync( String.format( @@ -209,7 +232,7 @@ public static void quickstart(String projectId, String location) { .deleteAspectTypeAsync( String.format("projects/%s/locations/global/aspectTypes/%s", projectId, aspectTypeId)) .get(); - System.out.println("Step 6: Successfully cleaned up resources"); + System.out.println("Step 7: Successfully cleaned up resources"); } catch (IOException | InterruptedException | ExecutionException e) { System.err.println("Error during quickstart execution: " + e); diff --git a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java index 1a59ec01c96..c177ada4ff5 100644 --- a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java +++ b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java @@ -77,7 +77,10 @@ public void testQuickstart() { "Step 5: Retrieved entry -> projects/%s/locations/%s/entryGroups" + "/dataplex-quickstart-entry-group/entries/dataplex-quickstart-entry", PROJECT_ID, LOCATION), - "Step 6: Successfully cleaned up resources"); + // Step 6 - result from Search + "locations/us-central1/entryGroups/dataplex-quickstart-entry-group" + + "/entries/dataplex-quickstart-entry", + "Step 7: Successfully cleaned up resources"); Quickstart.quickstart(PROJECT_ID, LOCATION); String output = bout.toString(); From 5d223f7bcfe2d3a4809f28f39dd6623b549a690c Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Mon, 18 Nov 2024 15:05:02 +0100 Subject: [PATCH 06/11] feat(dataplex): fix typos --- .../src/main/java/dataplex/Quickstart.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java index 0300bb80d52..ecd8d7f1b99 100644 --- a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -103,7 +103,7 @@ public static void quickstart(String projectId, String location) { client .createAspectTypeAsync( // Aspect Type is created in "global" location to highlight, that resources from - // "global" region can be attached Entry created in specific location + // "global" region can be attached to Entry created in specific location globalLocationName, aspectType, aspectTypeId) .get(); System.out.println("Step 1: Created aspect type -> " + createdAspectType.getName()); @@ -124,7 +124,7 @@ public static void quickstart(String projectId, String location) { EntryType createdEntryType = client // Entry Type is created in "global" location to highlight, that resources from - // "global" region can be attached Entry created in specific location + // "global" region can be attached to Entry created in specific location .createEntryTypeAsync(globalLocationName, entryType, entryTypeId) .get(); System.out.println("Step 2: Created entry type -> " + createdEntryType.getName()); @@ -180,18 +180,18 @@ public static void quickstart(String projectId, String location) { .setName(EntryName.of(projectId, location, entryGroupId, entryId).toString()) .setView(EntryView.FULL) .build(); - Entry retreivedEntry = client.getEntry(getEntryRequest); - System.out.println("Step 5: Retrieved entry -> " + retreivedEntry.getName()); - retreivedEntry + Entry retrievedEntry = client.getEntry(getEntryRequest); + System.out.println("Step 5: Retrieved entry -> " + retrievedEntry.getName()); + retrievedEntry .getAspectsMap() .values() .forEach( - retreivedAspect -> { + retrievedAspect -> { System.out.println("Retrieved aspect for entry:"); - System.out.println(" * aspect type -> " + retreivedAspect.getAspectType()); + System.out.println(" * aspect type -> " + retrievedAspect.getAspectType()); System.out.println( " * aspect field value -> " - + retreivedAspect + + retrievedAspect .getData() .getFieldsMap() .get("example_field") @@ -214,7 +214,7 @@ public static void quickstart(String projectId, String location) { .map(SearchEntriesResult::getDataplexEntry) .collect(Collectors.toList()); System.out.println("Entries found in Search:"); - // Please note in log outputs that Entry Group and Entry Type are also represented as Entries + // Please note in output that Entry Group and Entry Type are also represented as Entries entriesFromSearch.forEach( entryFromSearch -> System.out.println(" * " + entryFromSearch.getName())); From 70222b4c8e8cb2500ebea6ee7b5909ba0a9b1d87 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Fri, 22 Nov 2024 11:49:29 +0100 Subject: [PATCH 07/11] feat(dataplex): sleep execution to allow resources to propagate --- dataplex/quickstart/src/main/java/dataplex/Quickstart.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java index ecd8d7f1b99..61d3940cb60 100644 --- a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -140,6 +140,8 @@ public static void quickstart(String projectId, String location) { System.out.println("Step 3: Created entry group -> " + createdEntryGroup.getName()); // 4) Create Entry + // Wait 10 second to allow previously created resources to propagate + Thread.sleep(10000); Entry entry = Entry.newBuilder() .setEntryType( From 92f51f3c0722b9e7f5ee63eaf6f34da09c1f557c Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Fri, 22 Nov 2024 11:49:43 +0100 Subject: [PATCH 08/11] feat(dataplex): refactor test --- .../quickstart/src/test/java/dataplex/QuickstartIT.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java index c177ada4ff5..14ae7d46159 100644 --- a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java +++ b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java @@ -40,13 +40,8 @@ private static String requireProjectIdEnvVar() { } @BeforeClass - public static void checkRequirements() { - requireProjectIdEnvVar(); - } - - @BeforeClass - // Set-up code that will be executed before all tests public static void setUp() { + requireProjectIdEnvVar(); // Re-direct print stream to capture logging bout = new ByteArrayOutputStream(); originalPrintStream = System.out; @@ -89,7 +84,6 @@ public void testQuickstart() { } @AfterClass - // Clean-up code that will be executed after all tests public static void tearDown() { // Restore print statements System.setOut(originalPrintStream); From 0a9e8e1223bed6b8993a85a95a20fce2e502e899 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Fri, 22 Nov 2024 12:12:53 +0100 Subject: [PATCH 09/11] feat(dataplex): Move variables declaration to pass lint --- .../quickstart/src/main/java/dataplex/Quickstart.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java index 61d3940cb60..1d87679b80f 100644 --- a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -58,15 +58,11 @@ public static void quickstart(String projectId, String location) { // once, and can be reused for multiple requests. try (CatalogServiceClient client = CatalogServiceClient.create()) { // 0) Prepare variables used in following steps - String aspectTypeId = "dataplex-quickstart-aspect-type"; - String aspectKey = String.format("%s.global.%s", projectId, aspectTypeId); - String entryTypeId = "dataplex-quickstart-entry-type"; - String entryGroupId = "dataplex-quickstart-entry-group"; - String entryId = "dataplex-quickstart-entry"; LocationName globalLocationName = LocationName.of(projectId, "global"); LocationName specificLocationName = LocationName.of(projectId, location); // 1) Create Aspect Type that will be attached to Entry Type + String aspectTypeId = "dataplex-quickstart-aspect-type"; AspectType.MetadataTemplate aspectField = AspectType.MetadataTemplate.newBuilder() // The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$ @@ -109,6 +105,7 @@ public static void quickstart(String projectId, String location) { System.out.println("Step 1: Created aspect type -> " + createdAspectType.getName()); // 2) Create Entry Type, of which type Entry will be created + String entryTypeId = "dataplex-quickstart-entry-type"; EntryType entryType = EntryType.newBuilder() .setDescription("entry type for dataplex quickstart") @@ -130,6 +127,7 @@ public static void quickstart(String projectId, String location) { System.out.println("Step 2: Created entry type -> " + createdEntryType.getName()); // 3) Create Entry Group in which Entry will be located + String entryGroupId = "dataplex-quickstart-entry-group"; EntryGroup entryGroup = EntryGroup.newBuilder().setDescription("entry group for dataplex quickstart").build(); EntryGroup createdEntryGroup = @@ -142,6 +140,8 @@ public static void quickstart(String projectId, String location) { // 4) Create Entry // Wait 10 second to allow previously created resources to propagate Thread.sleep(10000); + String entryId = "dataplex-quickstart-entry"; + String aspectKey = String.format("%s.global.%s", projectId, aspectTypeId); Entry entry = Entry.newBuilder() .setEntryType( From 888e3ff17567f657b39201f4b535be394f2ffabb Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Fri, 22 Nov 2024 12:13:27 +0100 Subject: [PATCH 10/11] feat(dataplex): Enforce resource deletion before and after tests --- .../src/test/java/dataplex/QuickstartIT.java | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java index 14ae7d46159..6a3e6ce6299 100644 --- a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java +++ b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java @@ -19,7 +19,9 @@ import static com.google.common.truth.Truth.assertThat; import static junit.framework.TestCase.assertNotNull; +import com.google.cloud.dataplex.v1.CatalogServiceClient; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.PrintStream; import java.util.List; import org.junit.AfterClass; @@ -39,9 +41,45 @@ private static String requireProjectIdEnvVar() { return value; } + private static void forceCleanResources() throws IOException { + try (CatalogServiceClient client = CatalogServiceClient.create()) { + try { + client + .deleteEntryGroupAsync( + String.format( + "projects/%s/locations/%s/entryGroups/%s", + PROJECT_ID, LOCATION, "dataplex-quickstart-entry-group")) + .get(); + } catch (Exception e) { + // Pass, no resource to delete + } + try { + client + .deleteEntryTypeAsync( + String.format( + "projects/%s/locations/global/entryTypes/%s", + PROJECT_ID, "dataplex-quickstart-entry-type")) + .get(); + } catch (Exception e) { + // Pass, no resource to delete + } + try { + client + .deleteAspectTypeAsync( + String.format( + "projects/%s/locations/global/aspectTypes/%s", + PROJECT_ID, "dataplex-quickstart-aspect-type")) + .get(); + } catch (Exception e) { + // Pass, no resource to delete + } + } + } + @BeforeClass - public static void setUp() { + public static void setUp() throws IOException { requireProjectIdEnvVar(); + forceCleanResources(); // Re-direct print stream to capture logging bout = new ByteArrayOutputStream(); originalPrintStream = System.out; @@ -84,7 +122,8 @@ public void testQuickstart() { } @AfterClass - public static void tearDown() { + public static void tearDown() throws IOException { + forceCleanResources(); // Restore print statements System.setOut(originalPrintStream); bout.reset(); From e4efcff6f7cd69a74d57e2c66bf8e5fa69601240 Mon Sep 17 00:00:00 2001 From: Jacek Spalinski Date: Fri, 22 Nov 2024 12:46:03 +0100 Subject: [PATCH 11/11] feat(dataplex): Pass resources' IDs as method arguments --- .../src/main/java/dataplex/Quickstart.java | 19 ++++--- .../src/test/java/dataplex/QuickstartIT.java | 50 +++++++++---------- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java index 1d87679b80f..16afff3660d 100644 --- a/dataplex/quickstart/src/main/java/dataplex/Quickstart.java +++ b/dataplex/quickstart/src/main/java/dataplex/Quickstart.java @@ -46,14 +46,25 @@ public static void main(String[] args) { String projectId = "MY_PROJECT_ID"; // Available locations: https://cloud.google.com/dataplex/docs/locations String location = "MY_LOCATION"; + // Variables below can be replaced with custom values or defaults can be kept + String aspectTypeId = "dataplex-quickstart-aspect-type"; + String entryTypeId = "dataplex-quickstart-entry-type"; + String entryGroupId = "dataplex-quickstart-entry-group"; + String entryId = "dataplex-quickstart-entry"; - quickstart(projectId, location); + quickstart(projectId, location, aspectTypeId, entryTypeId, entryGroupId, entryId); } // Method to demonstrate lifecycle of different Dataplex resources and their interactions. // Method creates Aspect Type, Entry Type, Entry Group and Entry, retrieves Entry // and cleans up created resources. - public static void quickstart(String projectId, String location) { + public static void quickstart( + String projectId, + String location, + String aspectTypeId, + String entryTypeId, + String entryGroupId, + String entryId) { // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. try (CatalogServiceClient client = CatalogServiceClient.create()) { @@ -62,7 +73,6 @@ public static void quickstart(String projectId, String location) { LocationName specificLocationName = LocationName.of(projectId, location); // 1) Create Aspect Type that will be attached to Entry Type - String aspectTypeId = "dataplex-quickstart-aspect-type"; AspectType.MetadataTemplate aspectField = AspectType.MetadataTemplate.newBuilder() // The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$ @@ -105,7 +115,6 @@ public static void quickstart(String projectId, String location) { System.out.println("Step 1: Created aspect type -> " + createdAspectType.getName()); // 2) Create Entry Type, of which type Entry will be created - String entryTypeId = "dataplex-quickstart-entry-type"; EntryType entryType = EntryType.newBuilder() .setDescription("entry type for dataplex quickstart") @@ -127,7 +136,6 @@ public static void quickstart(String projectId, String location) { System.out.println("Step 2: Created entry type -> " + createdEntryType.getName()); // 3) Create Entry Group in which Entry will be located - String entryGroupId = "dataplex-quickstart-entry-group"; EntryGroup entryGroup = EntryGroup.newBuilder().setDescription("entry group for dataplex quickstart").build(); EntryGroup createdEntryGroup = @@ -140,7 +148,6 @@ public static void quickstart(String projectId, String location) { // 4) Create Entry // Wait 10 second to allow previously created resources to propagate Thread.sleep(10000); - String entryId = "dataplex-quickstart-entry"; String aspectKey = String.format("%s.global.%s", projectId, aspectTypeId); Entry entry = Entry.newBuilder() diff --git a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java index 6a3e6ce6299..9c2835cdd8f 100644 --- a/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java +++ b/dataplex/quickstart/src/test/java/dataplex/QuickstartIT.java @@ -24,15 +24,21 @@ import java.io.IOException; import java.io.PrintStream; import java.util.List; +import java.util.UUID; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; public class QuickstartIT { + private static final String ID = UUID.randomUUID().toString().substring(0, 8); private static final String LOCATION = "us-central1"; private static final String PROJECT_ID = requireProjectIdEnvVar(); private static ByteArrayOutputStream bout; private static PrintStream originalPrintStream; + private static final String ASPECT_TYPE_ID = "quickstart-aspect-type-" + ID; + private static final String ENTRY_TYPE_ID = "quickstart-entry-type-" + ID; + private static final String ENTRY_GROUP_ID = "quickstart-entry-group-" + ID; + private static final String ENTRY_ID = "quickstart-entry-" + ID; private static String requireProjectIdEnvVar() { String value = System.getenv("GOOGLE_CLOUD_PROJECT"); @@ -48,7 +54,7 @@ private static void forceCleanResources() throws IOException { .deleteEntryGroupAsync( String.format( "projects/%s/locations/%s/entryGroups/%s", - PROJECT_ID, LOCATION, "dataplex-quickstart-entry-group")) + PROJECT_ID, LOCATION, ENTRY_GROUP_ID)) .get(); } catch (Exception e) { // Pass, no resource to delete @@ -57,8 +63,7 @@ private static void forceCleanResources() throws IOException { client .deleteEntryTypeAsync( String.format( - "projects/%s/locations/global/entryTypes/%s", - PROJECT_ID, "dataplex-quickstart-entry-type")) + "projects/%s/locations/global/entryTypes/%s", PROJECT_ID, ENTRY_TYPE_ID)) .get(); } catch (Exception e) { // Pass, no resource to delete @@ -67,8 +72,7 @@ private static void forceCleanResources() throws IOException { client .deleteAspectTypeAsync( String.format( - "projects/%s/locations/global/aspectTypes/%s", - PROJECT_ID, "dataplex-quickstart-aspect-type")) + "projects/%s/locations/global/aspectTypes/%s", PROJECT_ID, ASPECT_TYPE_ID)) .get(); } catch (Exception e) { // Pass, no resource to delete @@ -77,9 +81,8 @@ private static void forceCleanResources() throws IOException { } @BeforeClass - public static void setUp() throws IOException { + public static void setUp() { requireProjectIdEnvVar(); - forceCleanResources(); // Re-direct print stream to capture logging bout = new ByteArrayOutputStream(); originalPrintStream = System.out; @@ -91,31 +94,28 @@ public void testQuickstart() { List expectedLogs = List.of( String.format( - "Step 1: Created aspect type -> projects/%s/locations/global/aspectTypes" - + "/dataplex-quickstart-aspect-type", - PROJECT_ID), + "Step 1: Created aspect type -> projects/%s/locations/global/aspectTypes/%s", + PROJECT_ID, ASPECT_TYPE_ID), String.format( - "Step 2: Created entry type -> projects/%s/locations/global/entryTypes" - + "/dataplex-quickstart-entry-type", - PROJECT_ID), + "Step 2: Created entry type -> projects/%s/locations/global/entryTypes/%s", + PROJECT_ID, ENTRY_TYPE_ID), String.format( - "Step 3: Created entry group -> projects/%s/locations/%s/entryGroups" - + "/dataplex-quickstart-entry-group", - PROJECT_ID, LOCATION), + "Step 3: Created entry group -> projects/%s/locations/%s/entryGroups/%s", + PROJECT_ID, LOCATION, ENTRY_GROUP_ID), String.format( - "Step 4: Created entry -> projects/%s/locations/%s/entryGroups" - + "/dataplex-quickstart-entry-group/entries/dataplex-quickstart-entry", - PROJECT_ID, LOCATION), + "Step 4: Created entry -> projects/%s/locations/%s/entryGroups/%s/entries/%s", + PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID), String.format( - "Step 5: Retrieved entry -> projects/%s/locations/%s/entryGroups" - + "/dataplex-quickstart-entry-group/entries/dataplex-quickstart-entry", - PROJECT_ID, LOCATION), + "Step 5: Retrieved entry -> projects/%s/locations/%s/entryGroups/%s/entries/%s", + PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID), // Step 6 - result from Search - "locations/us-central1/entryGroups/dataplex-quickstart-entry-group" - + "/entries/dataplex-quickstart-entry", + String.format( + "projects/%s/locations/%s/entryGroups/%s/entries/%s", + PROJECT_ID, LOCATION, ENTRY_GROUP_ID, ENTRY_ID), "Step 7: Successfully cleaned up resources"); - Quickstart.quickstart(PROJECT_ID, LOCATION); + Quickstart.quickstart( + PROJECT_ID, LOCATION, ASPECT_TYPE_ID, ENTRY_TYPE_ID, ENTRY_GROUP_ID, ENTRY_ID); String output = bout.toString(); expectedLogs.forEach(expectedLog -> assertThat(output).contains(expectedLog));