Upgrade Beam to 2.53.0

build.gradle

@@ -18,7 +18,7 @@ plugins {
     id "java"
     id "java-library"
     id "com.google.protobuf" version "0.9.4"
-    id "com.diffplug.spotless" version "6.2.2"
+    id "com.diffplug.spotless" version "6.25.0"
     id "idea"
     id "jacoco"
     id "com.github.johnrengelman.shadow" version "8.1.1"
@@ -32,17 +32,18 @@ repositories {
     }
 }
 
-def protoVersion = "3.24.4"
-def beamVersion = "2.51.0"
-def hadoopVersion = "3.3.6"
 def autoValueVersion = "1.10.4"
+def beamVersion = "2.53.0"
 def floggerVersion = "0.8"
+def hadoopVersion = "3.3.6"
+def jacksonVersion = "2.16.1"
+def protoVersion = "3.25.1"
 
 
 dependencies {
 
     // Protobuf deps
-    compileOnly "com.google.api.grpc:proto-google-common-protos:2.26.0"
+    compileOnly "com.google.api.grpc:proto-google-common-protos:2.31.0"
 
     implementation "com.google.protobuf:protobuf-java:${protoVersion}"
     implementation "com.google.protobuf:protobuf-java-util:${protoVersion}"
@@ -54,8 +55,9 @@ dependencies {
     testAnnotationProcessor "com.google.auto.value:auto-value:${autoValueVersion}"
 
     // Commons library deps
-    implementation 'com.google.guava:guava:32.1.3-jre'
-    implementation "org.apache.commons:commons-lang3:3.13.0"
+    implementation 'com.google.guava:guava:33.0.0-jre'
+    implementation "org.apache.commons:commons-lang3:3.14.0"
+    implementation "commons-io:commons-io:2.15.1"
     implementation "org.apache.httpcomponents:httpclient:4.5.14"
 
     // CSV Library
@@ -64,7 +66,7 @@ dependencies {
     // Google Services deps
     implementation "com.google.cloud:google-cloud-dlp:3.32.0"
     implementation "com.google.cloud:google-cloud-datacatalog:1.34.0"
-    implementation "com.google.cloud:google-cloud-kms:2.31.0"
+    implementation "com.google.cloud:google-cloud-kms:2.36.0"
     implementation "com.google.cloud:google-cloud-secretmanager:2.28.0"
 
     // Google Tink for encryption
@@ -74,23 +76,22 @@ dependencies {
     // JSON related libs
     implementation "com.github.wnameless.json:json-flattener:0.11.1"
     implementation "com.jayway.jsonpath:json-path:2.8.0"
-    implementation "com.fasterxml.jackson.core:jackson-core:2.13.1"
-    implementation "com.fasterxml.jackson.core:jackson-databind:2.13.1"
-
-    // Hadoop files (Avro + Parquet)
-    implementation "org.apache.avro:avro:1.11.3"
-    implementation "org.apache.parquet:parquet-avro:1.13.1"
-    implementation("org.apache.hadoop:hadoop-common:${hadoopVersion}"){
-        exclude group: "org.slf4j", module: "slf4j-log4j12"
-    }
+    implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+    implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
 
     // Logging framework
     implementation "com.google.flogger:flogger:${floggerVersion}"
     implementation "com.google.flogger:google-extensions:${floggerVersion}"
     runtimeOnly "com.google.flogger:flogger-system-backend:${floggerVersion}"
 
+    //Hadoop Common
+    runtimeOnly("org.apache.hadoop:hadoop-common:${hadoopVersion}"){
+        exclude group: "org.slf4j", module: "slf4j-log4j12"
+    }
+
     // Apache Beam deps
     implementation "org.apache.beam:beam-sdks-java-core:${beamVersion}"
+    implementation "org.apache.beam:beam-sdks-java-extensions-avro:${beamVersion}"
     implementation "org.apache.beam:beam-sdks-java-extensions-sorter:${beamVersion}"
     implementation "org.apache.beam:beam-sdks-java-io-jdbc:${beamVersion}"
     implementation "org.apache.beam:beam-sdks-java-io-google-cloud-platform:${beamVersion}"
@@ -127,7 +128,7 @@ dependencies {
     runtimeOnly "org.postgresql:postgresql:42.6.0"
 
     //checker-framework
-    compileOnly "org.checkerframework:checker-qual:3.39.0"
+    compileOnly "org.checkerframework:checker-qual:3.42.0"
 
     // Testing Dependencies
     testImplementation "junit:junit:4.13.2"

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/com/google/cloud/solutions/autotokenize/common/ByteValueConverter.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import com.google.privacy.dlp.v2.Value;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;

src/main/java/com/google/cloud/solutions/autotokenize/common/FlatRecordConvertFn.java

@@ -16,11 +16,10 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.FlatRecord;
 import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
-import org.apache.beam.sdk.schemas.utils.AvroUtils;
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.Row;

src/main/java/com/google/cloud/solutions/autotokenize/common/IdentityFn.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import org.apache.beam.sdk.transforms.SerializableFunction;
 
 /** Identity function: Returns the same value as input. */

src/main/java/com/google/cloud/solutions/autotokenize/common/InspectionReportFileWriter.java

@@ -79,10 +79,10 @@ public PDone expand(PCollection<InspectionReport> inspectionReport) {
                 Contextful.fn(
                     colName ->
                         defaultNaming(
-                            /*prefix=*/ String.format(
+                            /* prefix= */ String.format(
                                     "col-%s", colName.replaceAll("[\\.\\$\\[\\]]+", "-"))
                                 .replaceAll("[-]+", "-"),
-                            /*suffix=*/ ".json")))
+                            /* suffix= */ ".json")))
             .to(reportLocation()));
 
     return PDone.in(inspectionReport.getPipeline());

src/main/java/com/google/cloud/solutions/autotokenize/common/RecordFlattener.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.FlatRecord;
 
 public interface RecordFlattener<T> {

src/main/java/com/google/cloud/solutions/autotokenize/common/RecordNester.java

@@ -16,12 +16,11 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import com.google.auto.value.AutoValue;
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.FlatRecord;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericRecord;
-import org.apache.beam.sdk.coders.AvroGenericCoder;
+import org.apache.beam.sdk.extensions.avro.coders.AvroGenericCoder;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SimpleFunction;

src/main/java/com/google/cloud/solutions/autotokenize/common/SecretsClient.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
 import com.google.cloud.secretmanager.v1.stub.SecretManagerServiceStub;
 import com.google.common.flogger.GoogleLogger;

src/main/java/com/google/cloud/solutions/autotokenize/common/SourceNames.java

@@ -34,7 +34,7 @@ public static SourceNames forType(SourceType sourceType) {
   }
 
   public String asCamelCase() {
-    return CaseUtils.toCamelCase(sourceType.name(), /*capitalizeFirstLetter=*/ true, '_');
+    return CaseUtils.toCamelCase(sourceType.name(), /* capitalizeFirstLetter= */ true, '_');
   }
 
   @Override

src/main/java/com/google/cloud/solutions/autotokenize/common/TransformingReader.java

@@ -36,8 +36,8 @@
 import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.extensions.avro.io.AvroIO;
 import org.apache.beam.sdk.extensions.protobuf.ProtoCoder;
-import org.apache.beam.sdk.io.AvroIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices;
 import org.apache.beam.sdk.io.jdbc.JdbcIO;

src/main/java/com/google/cloud/solutions/autotokenize/common/UnFlattenKvFn.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.common;
 
-
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.values.KV;
 

src/main/java/com/google/cloud/solutions/autotokenize/datacatalog/DataCatalogWriter.java

@@ -291,11 +291,11 @@ private void updateTag(Tag tag) {
       private ImmutableMap<String, Tag> getExistingTags() {
         return StreamSupport.stream(
                 catalogClient.listTags(targetEntryId).iteratePages().spliterator(),
-                /*parallel=*/ false)
+                /* parallel= */ false)
             .map(ListTagsPage::getValues)
             .flatMap(
                 tagsIterator ->
-                    StreamSupport.stream(tagsIterator.spliterator(), /*parallel=*/ false))
+                    StreamSupport.stream(tagsIterator.spliterator(), /* parallel= */ false))
             .filter(tag -> inspectionTagTemplateId().equals(tag.getTemplate()))
             .collect(toImmutableMap(Tag::getColumn, identity()));
       }

src/main/java/com/google/cloud/solutions/autotokenize/dlp/BatchAccumulator.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import com.google.common.collect.ImmutableList;
 import java.util.Iterator;
 import org.checkerframework.checker.nullness.qual.Nullable;

src/main/java/com/google/cloud/solutions/autotokenize/dlp/BatchAccumulatorFactory.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import java.io.Serializable;
 
 public interface BatchAccumulatorFactory<InputT, OutputT> extends Serializable {

src/main/java/com/google/cloud/solutions/autotokenize/dlp/BatchColumnsForDlp.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import com.google.common.flogger.GoogleLogger;
 import com.google.privacy.dlp.v2.FieldId;
 import com.google.privacy.dlp.v2.Table;

src/main/java/com/google/cloud/solutions/autotokenize/dlp/DlpBatchInspectFactory.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
 import com.google.privacy.dlp.v2.InfoType;

src/main/java/com/google/cloud/solutions/autotokenize/dlp/DlpClientFactory.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import com.google.api.gax.rpc.FixedHeaderProvider;
 import com.google.cloud.dlp.v2.DlpServiceClient;
 import com.google.cloud.dlp.v2.DlpServiceSettings;

src/main/java/com/google/cloud/solutions/autotokenize/dlp/DlpIdentify.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import com.google.auto.value.AutoValue;
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.ColumnInformation;
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.InfoTypeInformation;

src/main/java/com/google/cloud/solutions/autotokenize/dlp/DlpUtil.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.dlp;
 
-
 import org.checkerframework.checker.nullness.qual.NonNull;
 
 /** Implements simple DLP API related utility functions. */

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/AesEcbStringValueTokenizer.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.common.io.BaseEncoding;
 import com.google.privacy.dlp.v2.Value;
 import java.nio.charset.StandardCharsets;

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/ClearTextKeySetExtractor.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/DaeadEncryptingValueTokenizer.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.crypto.tink.DeterministicAead;
 import com.google.privacy.dlp.v2.Value;
 import com.google.protobuf.InvalidProtocolBufferException;

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/DaeadEncryptingValueTokenizerFactory.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.KeyMaterialType;
 import com.google.crypto.tink.DeterministicAead;
 import com.google.crypto.tink.daead.DeterministicAeadConfig;

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/FlatRecordTokenizer.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.FlatRecord;
 
 /** A Record processor that can tokenize specific or all values of a {@link FlatRecord} */

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/GcpKmsClearTextKeySetExtractor.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.crypto.tink.CleartextKeysetHandle;
 import com.google.crypto.tink.JsonKeysetReader;
 import com.google.crypto.tink.JsonKeysetWriter;

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/ValueTokenizer.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.privacy.dlp.v2.Value;
 
 /**

src/main/java/com/google/cloud/solutions/autotokenize/encryptors/ValueTokenizerFactory.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.encryptors;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.KeyMaterialType;
 import java.io.Serializable;
 

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/AutoInspectAndTokenizeOptions.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.pipeline;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages;
 import java.util.List;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/DlpInspectionOptions.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.pipeline;
 
-
 import java.util.List;
 import org.apache.beam.sdk.options.Default;
 

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/DlpInspectionPipeline.java

@@ -154,10 +154,10 @@ Pipeline makePipeline() {
                     Contextful.fn(
                         colName ->
                             defaultNaming(
-                                /*prefix=*/ String.format(
+                                /* prefix= */ String.format(
                                         "col-%s", colName.replaceAll("[\\.\\$\\[\\]]+", "-"))
                                     .replaceAll("[-]+", "-"),
-                                /*suffix=*/ ".json")))
+                                /* suffix= */ ".json")))
                 .to(options.getReportLocation() + "/error"));
 
     var inspectionReport =

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/EncryptionPipeline.java

@@ -50,10 +50,10 @@
 import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.io.AvroIO;
+import org.apache.beam.sdk.extensions.avro.io.AvroIO;
+import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.schemas.utils.AvroUtils;
 import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TupleTag;

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/EncryptionPipelineOptions.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.pipeline;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.KeyMaterialType;
 import java.util.List;
 import org.apache.beam.sdk.options.Default;

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/KeyMaterialTypeFactory.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.pipeline;
 
-
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.KeyMaterialType;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.PipelineOptions;

src/main/java/com/google/cloud/solutions/autotokenize/pipeline/ValueEncryptionTransform.java

@@ -16,7 +16,6 @@
 
 package com.google.cloud.solutions.autotokenize.pipeline;
 
-
 import com.google.auto.value.AutoValue;
 import com.google.cloud.solutions.autotokenize.AutoTokenizeMessages.FlatRecord;
 import com.google.cloud.solutions.autotokenize.encryptors.EncryptingFlatRecordTokenizer;