[CYB-207] cybersec toolkit supports CSA 1.12.0 with CDP 7.1.9

cloudera · Mar 22, 2024 · d82f591 · d82f591
1 parent bd26382
commit d82f591
Show file tree

Hide file tree

Showing 25 changed files with 228 additions and 73 deletions.
diff --git a/flink-cyber/caracal-generator/pom.xml b/flink-cyber/caracal-generator/pom.xml
@@ -33,6 +33,13 @@
             <artifactId>flink-connector-kafka</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients</artifactId>
+            <scope>provided</scope>
+            <version>${flink.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>com.hortonworks.smm</groupId>
             <artifactId>monitoring-interceptors</artifactId>

diff --git a/flink-cyber/caracal-parser/pom.xml b/flink-cyber/caracal-parser/pom.xml
@@ -126,7 +126,7 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
         </dependency>
 
         <dependency>

diff --git a/flink-cyber/flink-alert-scoring/pom.xml b/flink-cyber/flink-alert-scoring/pom.xml
@@ -95,6 +95,11 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-statebackend-rocksdb</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>test</scope>
+        </dependency>
 
     </dependencies>
     <build>

diff --git a/flink-cyber/flink-commands/scoring-commands/pom.xml b/flink-cyber/flink-commands/scoring-commands/pom.xml
@@ -55,6 +55,14 @@
             <version>${jackson.datatype.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
         <!-- Logging and Metrics -->
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -85,6 +93,10 @@
             <artifactId>avro</artifactId>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-java</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -123,6 +135,12 @@
                                     </excludes>
                                 </filter>
                             </filters>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google.</pattern>
+                                    <shadedPattern>com.cloudera.cyber.com.google.</shadedPattern>
+                                </relocation>
+                            </relocations>
                             <transformers>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                     <mainClass>com.cloudera.cyber.scoring.ScoringJobKafka</mainClass>

diff --git a/flink-cyber/flink-common/pom.xml b/flink-cyber/flink-common/pom.xml
@@ -49,12 +49,17 @@
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
         </dependency>
         <dependency>
             <groupId>com.hortonworks.registries</groupId>
             <artifactId>schema-registry-serdes</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-connector-base</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>

diff --git a/flink-cyber/flink-dedupe/pom.xml b/flink-cyber/flink-dedupe/pom.xml
@@ -45,8 +45,9 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
         </dependency>
+
         <dependency>
             <groupId>com.hortonworks.registries</groupId>
             <artifactId>schema-registry-serdes</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml
@@ -90,6 +90,11 @@
             <classifier>tests</classifier>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.hortonworks.smm</groupId>
             <artifactId>monitoring-interceptors</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml
@@ -156,6 +156,11 @@
             <version>${commons-validator.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml
@@ -76,9 +76,10 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>compile</scope>
         </dependency>
+
         <dependency>
             <groupId>com.hortonworks.registries</groupId>
             <artifactId>schema-registry-serdes</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml
@@ -39,6 +39,14 @@
             <artifactId>lombok</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
 
     </dependencies>
 

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml
@@ -56,14 +56,15 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>com.hortonworks.registries</groupId>
             <artifactId>schema-registry-serdes</artifactId>
         </dependency>
+
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-cyber-api</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml
@@ -50,7 +50,7 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml
@@ -51,10 +51,15 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-cyber-api</artifactId>

diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml
@@ -51,7 +51,7 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
             <scope>provided</scope>
         </dependency>
 

diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml b/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml
@@ -57,22 +57,42 @@
             <version>${project.parent.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-runtime</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
 
         <!-- Flink Dependency -->
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.esotericsoftware.kryo</groupId>
+                    <artifactId>kryo</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-table-api-java-bridge</artifactId>
+            <artifactId>flink-table-api-java</artifactId>
             <scope>provided</scope>
+            <version>${flink.version}</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+            <artifactId>flink-table-planner-loader</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -89,7 +109,7 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
         </dependency>
 
         <dependency>
@@ -101,6 +121,10 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-json</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
 
         <!-- Hive dependencies -->
         <dependency>
@@ -350,6 +374,10 @@
             <version>2.2.0</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-api-java-bridge</artifactId>
+        </dependency>
 
     </dependencies>
 
@@ -360,7 +388,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.4.1</version>
+                <version>${maven-shade-plugin.version}</version>
                 <executions>
                     <!-- Run shade goal on package phase -->
                     <execution>

diff --git a/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml b/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml
@@ -100,7 +100,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>${maven-shade-plugin.version}</version>
                 <executions>
                     <!-- Run shade goal on package phase -->
                     <execution>

diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml b/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml
@@ -85,6 +85,11 @@
             <artifactId>flink-connector-kafka</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils</artifactId>
@@ -100,7 +105,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>${maven-shade-plugin.version}</version>
                 <executions>
                     <!-- Run shade goal on package phase -->
                     <execution>
@@ -130,6 +135,12 @@
                                     </excludes>
                                 </filter>
                             </filters>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google.</pattern>
+                                    <shadedPattern>com.cloudera.cyber.com.google.</shadedPattern>
+                                </relocation>
+                            </relocations>
                             <transformers>
                                 <transformer
                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

diff --git a/flink-cyber/flink-indexing/pom.xml b/flink-cyber/flink-indexing/pom.xml
@@ -27,7 +27,6 @@
     <modules>
         <module>flink-indexing-parquet</module>
         <module>flink-indexing-solr</module>
-        <module>flink-indexing-elastic</module>
         <module>flink-indexing-search</module>
         <module>flink-indexing-hive</module>
        <!-- <module>flink-indexing-hive-partitions</module> -->

diff --git a/flink-cyber/flink-profiler-java/pom.xml b/flink-cyber/flink-profiler-java/pom.xml
@@ -37,6 +37,14 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-hbase-${flink.hbase.version}</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.hortonworks.registries</groupId>
+            <artifactId>schema-registry-serdes</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.cloudera.cyber</groupId>
             <artifactId>flink-profiler</artifactId>

diff --git a/flink-cyber/flink-sessions/pom.xml b/flink-cyber/flink-sessions/pom.xml
@@ -35,7 +35,7 @@
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-cloudera-registry</artifactId>
+            <artifactId>flink-connector-cloudera-registry</artifactId>
         </dependency>
 
         <dependency>

diff --git a/flink-cyber/flink-stellar/pom.xml b/flink-cyber/flink-stellar/pom.xml
@@ -35,7 +35,7 @@
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <commons.config.version>1.10</commons.config.version>
         <guava_version>${guava.version}</guava_version>
-        <curator.version>4.3.0.${cdh.version}</curator.version>
+        <curator.version>5.4.0.${cdh.version}</curator.version>
         <global_aesh_version>0.66.19</global_aesh_version>
         <global_jackson_version>2.7.4</global_jackson_version>
         <global_json_simple_version>1.1.1</global_json_simple_version>

diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml b/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml
@@ -27,7 +27,6 @@
         <!-- main dependency versions -->
         <antlr4.version>${antlr.version}</antlr4.version>
         <java-grok.version>0.1.9</java-grok.version>
-        <global_shade_version>3.2.0</global_shade_version>
     </properties>
 
     <dependencies>
@@ -194,7 +193,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>${global_shade_version}</version>
+                <version>${maven-shade-plugin.version}</version>
                 <configuration>
                     <createDependencyReducedPom>true</createDependencyReducedPom>
                     <artifactSet>