diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9e1f52f..cba6558c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file. - hdfs: Exclude YARN and Mapreduce projects from build ([#667]). - stackable-base: Mitigate CVE-2023-37920 by removing e-Tugra root certificates ([#673]). - hdfs: Exclude unused jars and mitigate snappy-java CVEs by bumping dependency ([#682]). +- druid: Build from source ([#684]). ### Changed @@ -83,6 +84,7 @@ All notable changes to this project will be documented in this file. [#678]: https://github.com/stackabletech/docker-images/pull/678 [#679]: https://github.com/stackabletech/docker-images/pull/679 [#682]: https://github.com/stackabletech/docker-images/pull/682 +[#684]: https://github.com/stackabletech/docker-images/pull/684 [#685]: https://github.com/stackabletech/docker-images/pull/685 [#688]: https://github.com/stackabletech/docker-images/pull/688 diff --git a/conf.py b/conf.py index b16294f97..b58964b6d 100644 --- a/conf.py +++ b/conf.py @@ -56,6 +56,7 @@ { "product": "26.0.0", "java-base": "11", + "java-devel": "11", "jackson_dataformat_xml": "2.10.5", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", @@ -64,6 +65,7 @@ { "product": "27.0.0", "java-base": "11", + "java-devel": "11", "jackson_dataformat_xml": "2.10.5", "stax2_api": "4.2.1", "woodstox_core": "6.2.1", @@ -76,6 +78,7 @@ # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class # java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) throws java.lang.ClassFormatError "java-base": "11", + "java-devel": "11", "jackson_dataformat_xml": "2.12.7", # from https://github.com/apache/druid/blob/b8201e31aa6b124049a61764309145baaad78db7/pom.xml#L100 "stax2_api": "4.2.2", "woodstox_core": "6.6.0", diff --git a/druid/Dockerfile b/druid/Dockerfile index fc0883633..f0381b30f 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -4,13 +4,69 @@ # Ignoring DL4006 globally because we inherit the SHELL from our base image # hadolint global ignore=DL3038,DL4006 -FROM stackable/image/java-base +# hadolint ignore=DL3006 +FROM stackable/image/java-devel as druid-builder ARG PRODUCT ARG JACKSON_DATAFORMAT_XML ARG STAX2_API ARG WOODSTOX_CORE ARG AUTHORIZER + +RUN microdnf update && \ + microdnf install \ + # Required to install pyyaml + python-pip && \ + microdnf clean all && \ + rm -rf /var/cache/yum && \ + # pyyaml is required for the compile Druid + pip install --no-cache-dir pyyaml==6.0.1 + +USER stackable +WORKDIR /stackable + +RUN curl --fail -L "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC . && \ + cd apache-druid-${PRODUCT}-src && \ + mvn clean install -Pdist -DskipTests -Dmaven.javadoc.skip=true && \ + tar -xzf /stackable/apache-druid-${PRODUCT}-src/distribution/target/apache-druid-${PRODUCT}-bin.tar.gz && \ + mv /stackable/apache-druid-${PRODUCT}-src/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT} && \ + rm -rf /stackable/apache-druid-${PRODUCT}-src \ + /stackable/apache-druid-${PRODUCT}/quickstart + + # Install the Prometheus emitter extension. This bundle contains the emitter and all jar dependencies. +RUN curl --fail "https://repo.stackable.tech/repository/packages/druid/druid-prometheus-emitter-${PRODUCT}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions && \ + # Install OPA authorizer extension. + curl --fail "https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions && \ + # Install jackson-dataformat-xml, stax2-api, and woodstox-core which are required for logging, and remove stax-ex. + rm /stackable/apache-druid-${PRODUCT}/lib/stax-ex-*.jar && \ + curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar \ + "https://repo.stackable.tech/repository/packages/jackson-dataformat-xml/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar" && \ + curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/stax2-api-${STAX2_API}.jar \ + "https://repo.stackable.tech/repository/packages/stax2-api/stax2-api-${STAX2_API}.jar" && \ + curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/woodstox-core-${WOODSTOX_CORE}.jar \ + "https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar" + +# For earlier versions this script removes the .class file that contains the +# vulnerable code. +# TODO: This can be restricted to target only versions which do not honor the environment +# varible that has been set above but this has not currently been implemented +COPY shared/log4shell.sh /bin +RUN /bin/log4shell.sh "/stackable/apache-druid-${PRODUCT}" + +# Ensure no vulnerable files are left over +# This will currently report vulnerable files being present, as it also alerts on +# SocketNode.class, which we do not remove with our scripts. +# Further investigation will be needed whether this should also be removed. +COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64 +COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64 +COPY shared/log4shell_scanner /bin/log4shell_scanner +RUN /bin/log4shell_scanner s "/stackable/apache-druid-${PRODUCT}" +# === + +# hadolint ignore=DL3006 +FROM stackable/image/java-base as final + +ARG PRODUCT ARG RELEASE LABEL name="Apache Druid" \ @@ -22,36 +78,27 @@ LABEL name="Apache Druid" \ description="This image is deployed by the Stackable Operator for Apache Druid." RUN microdnf update && \ - microdnf install \ - findutils \ - gzip \ - tar \ - zip && \ microdnf clean all && \ rm -rf /var/cache/yum USER stackable WORKDIR /stackable +COPY --from=druid-builder /stackable /stackable COPY --chown=stackable:stackable druid/stackable /stackable COPY --chown=stackable:stackable druid/licenses /licenses -RUN curl --fail -L https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-bin.tar.gz | tar -xzC . && \ - ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \ +RUN ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \ # Force to overwrite the existing 'run-druid' - ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid && \ - # Install the Prometheus emitter extension. This bundle contains the emitter and all jar dependencies. - curl --fail https://repo.stackable.tech/repository/packages/druid/druid-prometheus-emitter-${PRODUCT}.tar.gz | tar -xzC /stackable/druid/extensions && \ - # Install OPA authorizer extension. - curl --fail https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz | tar -xzC /stackable/druid/extensions && \ - # Install jackson-dataformat-xml, stax2-api, and woodstox-core which are required for logging, and remove stax-ex. - rm /stackable/druid/lib/stax-ex-*.jar && \ - curl --fail -L -o /stackable/druid/lib/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar \ - https://repo.stackable.tech/repository/packages/jackson-dataformat-xml/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar && \ - curl --fail -L -o /stackable/druid/lib/stax2-api-${STAX2_API}.jar \ - https://repo.stackable.tech/repository/packages/stax2-api/stax2-api-${STAX2_API}.jar && \ - curl --fail -L -o /stackable/druid/lib/woodstox-core-${WOODSTOX_CORE}.jar \ - https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar + ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid + +ENV PATH="${PATH}":/stackable/druid/bin + +# === +# Mitigation for CVE-2021-44228 (Log4Shell) +# This variable is supported as of Log4j version 2.10 and +# disables the vulnerable feature +ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true WORKDIR /stackable/druid CMD ["bin/run-druid", "coordinator", "conf/druid/cluster/master/coordinator-overlord/"] diff --git a/druid/upload_new_druid_version.sh b/druid/upload_new_druid_version.sh index b04107e5d..a3f9bb988 100755 --- a/druid/upload_new_druid_version.sh +++ b/druid/upload_new_druid_version.sh @@ -29,16 +29,16 @@ trap cleanup EXIT cd "$WORK_DIR" || exit -bin_file="apache-druid-${VERSION}-bin.tar.gz" +src_file="apache-druid-${VERSION}-src.tar.gz" echo "Downloading Druid (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${bin_file}" -curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${bin_file}.asc" -curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${bin_file}.sha512" +curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}" +curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}.asc" +curl --fail -LOs "https://archive.apache.org/dist/druid/${VERSION}/${src_file}.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksum" -if ! (sha512sum "${bin_file}" | cut -d " " -f 1 | diff -Z - "${bin_file}.sha512"); then +if ! (sha512sum "${src_file}" | cut -d " " -f 1 | diff -Z - "${src_file}.sha512"); then echo "ERROR: The SHA512 sum does not match" exit 1 fi @@ -46,16 +46,16 @@ fi echo "Validating signature" echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/spark/KEYS | gpg --import")' -if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then +if ! (gpg --verify "${src_file}.asc" "${src_file}" 2> /dev/null); then echo "ERROR: The signature could not be verified" exit 1 fi echo "Uploading everything to Nexus" EXIT_STATUS=0 -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? +curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/druid/' || EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed"