feat: Build Druid from source

CHANGELOG.md

@@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file.
 - hdfs: Exclude YARN and Mapreduce projects from build ([#667]).
 - stackable-base: Mitigate CVE-2023-37920 by removing e-Tugra root certificates ([#673]).
 - hdfs: Exclude unused jars and mitigate snappy-java CVEs by bumping dependency ([#682]).
+- druid: Build from source ([#684]).
 
 ### Changed
 
@@ -83,6 +84,7 @@ All notable changes to this project will be documented in this file.
 [#678]: https://github.com/stackabletech/docker-images/pull/678
 [#679]: https://github.com/stackabletech/docker-images/pull/679
 [#682]: https://github.com/stackabletech/docker-images/pull/682
+[#684]: https://github.com/stackabletech/docker-images/pull/684
 [#685]: https://github.com/stackabletech/docker-images/pull/685
 [#688]: https://github.com/stackabletech/docker-images/pull/688
 

conf.py

@@ -56,6 +56,7 @@
             {
                 "product": "26.0.0",
                 "java-base": "11",
+                "java-devel": "11",
                 "jackson_dataformat_xml": "2.10.5",
                 "stax2_api": "4.2.1",
                 "woodstox_core": "6.2.1",
@@ -64,6 +65,7 @@
             {
                 "product": "27.0.0",
                 "java-base": "11",
+                "java-devel": "11",
                 "jackson_dataformat_xml": "2.10.5",
                 "stax2_api": "4.2.1",
                 "woodstox_core": "6.2.1",
@@ -76,6 +78,7 @@
                 # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class
                 # java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) throws java.lang.ClassFormatError
                 "java-base": "11",
+                "java-devel": "11",
                 "jackson_dataformat_xml": "2.12.7",  # from https://github.com/apache/druid/blob/b8201e31aa6b124049a61764309145baaad78db7/pom.xml#L100
                 "stax2_api": "4.2.2",
                 "woodstox_core": "6.6.0",

druid/Dockerfile

@@ -4,15 +4,78 @@
 # Ignoring DL4006 globally because we inherit the SHELL from our base image
 # hadolint global ignore=DL3038,DL4006
 
-FROM stackable/image/java-base
+# hadolint ignore=DL3006
+FROM stackable/image/java-devel as druid-builder
 
 ARG PRODUCT
 ARG JACKSON_DATAFORMAT_XML
 ARG STAX2_API
 ARG WOODSTOX_CORE
 ARG AUTHORIZER
+
+RUN microdnf update && \
+    microdnf install \
+    # Required to install pyyaml
+    python-pip && \
+    microdnf clean all && \
+    rm -rf /var/cache/yum && \
+    # pyyaml is required for the compile Druid
+    pip install --no-cache-dir pyyaml==6.0.1
+
+USER stackable
+WORKDIR /stackable
+
+RUN curl --fail -L "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC . && \
+    cd apache-druid-${PRODUCT}-src && \
+    mvn clean install -Pdist -DskipTests -Dmaven.javadoc.skip=true && \
+    tar -xzf /stackable/apache-druid-${PRODUCT}-src/distribution/target/apache-druid-${PRODUCT}-bin.tar.gz && \
+    mv /stackable/apache-druid-${PRODUCT}-src/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT} && \
+    rm -rf /stackable/apache-druid-${PRODUCT}-src \
+    /stackable/apache-druid-${PRODUCT}/quickstart
+
+    # Install the Prometheus emitter extension. This bundle contains the emitter and all jar dependencies.
+RUN curl --fail "https://repo.stackable.tech/repository/packages/druid/druid-prometheus-emitter-${PRODUCT}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions && \
+    # Install OPA authorizer extension.
+    curl --fail "https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions && \
+    # Install jackson-dataformat-xml, stax2-api, and woodstox-core which are required for logging, and remove stax-ex.
+    rm /stackable/apache-druid-${PRODUCT}/lib/stax-ex-*.jar && \
+    curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar \
+        "https://repo.stackable.tech/repository/packages/jackson-dataformat-xml/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar" && \
+    curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/stax2-api-${STAX2_API}.jar \
+        "https://repo.stackable.tech/repository/packages/stax2-api/stax2-api-${STAX2_API}.jar" && \
+    curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/woodstox-core-${WOODSTOX_CORE}.jar \
+        "https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar"
+
+# For earlier versions this script removes the .class file that contains the
+# vulnerable code.
+# TODO: This can be restricted to target only versions which do not honor the environment
+#   varible that has been set above but this has not currently been implemented
+COPY shared/log4shell.sh /bin
+RUN /bin/log4shell.sh "/stackable/apache-druid-${PRODUCT}"
+
+# Ensure no vulnerable files are left over
+# This will currently report vulnerable files being present, as it also alerts on
+# SocketNode.class, which we do not remove with our scripts.
+# Further investigation will be needed whether this should also be removed.
+COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
+COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
+COPY shared/log4shell_scanner /bin/log4shell_scanner
+RUN /bin/log4shell_scanner s "/stackable/apache-druid-${PRODUCT}"
+# ===
+
+## Final Image
+# hadolint ignore=DL3006
+FROM stackable/image/java-base
+
+ARG PRODUCT
 ARG RELEASE
 
+# ===
+# Mitigation for CVE-2021-44228 (Log4Shell)
+# This variable is supported as of Log4j version 2.10 and
+# disables the vulnerable feature
+ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
+
 LABEL name="Apache Druid" \
       maintainer="info@stackable.tech" \
       vendor="Stackable GmbH" \
@@ -22,36 +85,19 @@
       description="This image is deployed by the Stackable Operator for Apache Druid."
 
 RUN microdnf update && \
-    microdnf install \
-    findutils \
-    gzip \
-    tar \
-    zip && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
 USER stackable
 WORKDIR /stackable
 
+COPY --from=druid-builder /stackable /stackable
 COPY --chown=stackable:stackable druid/stackable /stackable
 COPY --chown=stackable:stackable druid/licenses /licenses
 
-RUN curl --fail -L https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-bin.tar.gz | tar -xzC . && \
-    ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \
+RUN ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \
     # Force to overwrite the existing 'run-druid'
-    ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid && \
-    # Install the Prometheus emitter extension. This bundle contains the emitter and all jar dependencies.
-    curl --fail https://repo.stackable.tech/repository/packages/druid/druid-prometheus-emitter-${PRODUCT}.tar.gz | tar -xzC /stackable/druid/extensions && \
-    # Install OPA authorizer extension.
-    curl --fail https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz | tar -xzC /stackable/druid/extensions && \
-    # Install jackson-dataformat-xml, stax2-api, and woodstox-core which are required for logging, and remove stax-ex.
-    rm /stackable/druid/lib/stax-ex-*.jar && \
-    curl --fail -L -o /stackable/druid/lib/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar \
-        https://repo.stackable.tech/repository/packages/jackson-dataformat-xml/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar && \
-    curl --fail -L -o /stackable/druid/lib/stax2-api-${STAX2_API}.jar \
-        https://repo.stackable.tech/repository/packages/stax2-api/stax2-api-${STAX2_API}.jar && \
-    curl --fail -L -o /stackable/druid/lib/woodstox-core-${WOODSTOX_CORE}.jar \
-        https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar
+    ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
 
 WORKDIR /stackable/druid
 CMD ["bin/run-druid", "coordinator", "conf/druid/cluster/master/coordinator-overlord/"]

druid/upload_new_druid_version.sh

@@ -29,33 +29,33 @@ trap cleanup EXIT
 
 cd "$WORK_DIR" || exit
 
-bin_file="apache-druid-${VERSION}-bin.tar.gz"
+src_file="apache-druid-${VERSION}-src.tar.gz"
 
 echo "Downloading Druid (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
-curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${bin_file}"
-curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${bin_file}.asc"
-curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${bin_file}.sha512"
+curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}"
+curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}.asc"
+curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}.sha512"
 
 # It is probably redundant to check both the checksum and the signature but it's cheap and why not
 echo "Validating SHA512 Checksum"
-if ! (sha512sum "${bin_file}" | cut -d " " -f 1 | diff -Z - "${bin_file}.sha512"); then
+if ! (sha512sum "${src_file}" | cut -d " " -f 1 | diff -Z - "${src_file}.sha512"); then
   echo "ERROR: The SHA512 sum does not match"
   exit 1
 fi
 
 echo "Validating signature"
 echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/spark/KEYS | gpg --import")'
 
-if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then
+if ! (gpg --verify "${src_file}.asc" "${src_file}" 2> /dev/null); then
   echo "ERROR: The signature could not be verified"
   exit 1
 fi
 
 echo "Uploading everything to Nexus"
 EXIT_STATUS=0
-curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$?
-curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$?
-curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$?
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$?
 
 if [ $EXIT_STATUS -ne 0 ]; then
   echo "ERROR: Upload failed"