feat: Add log processing script to opa for decision logging (#695)

* add script to process logs from opa

* exit with error if parameters are not set

* jq for log filtering

* add changelog

* security fixes

* shellcheck linting

* fix changelog entry

* add errorhandling for jq command

* small code improvements

* add information to catch output

* read from env vars directly

CHANGELOG.md

@@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file.
 - stackable-base: Mitigate CVE-2023-37920 by removing e-Tugra root certificates ([#673]).
 - hdfs: Exclude unused jars and mitigate snappy-java CVEs by bumping dependency ([#682]).
 - druid: Build from source ([#684], [#696]).
+- opa: Add log processing script to opa for decision logging ([#695]).
 
 ### Changed
 
@@ -88,6 +89,7 @@ All notable changes to this project will be documented in this file.
 [#685]: https://github.com/stackabletech/docker-images/pull/685
 [#688]: https://github.com/stackabletech/docker-images/pull/688
 [#696]: https://github.com/stackabletech/docker-images/pull/696
+[#695]: https://github.com/stackabletech/docker-images/pull/695
 
 ## [24.3.0] - 2024-03-20
 

opa/Dockerfile

@@ -109,6 +109,9 @@ LABEL name="Open Policy Agent" \
       description="This image is deployed by the Stackable Operator for OPA."
 
 RUN microdnf update && \
+    microdnf install \
+    # Required for filtering logs
+    jq && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
@@ -121,4 +124,8 @@ COPY --from=opa-builder /opa/opa /stackable/opa/opa
 COPY --from=opa-bundle-builder --chown=stackable:stackable /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder
 COPY --from=multilog-builder --chown=stackable:stackable /daemontools/admin/daemontools/command/multilog /stackable/multilog
 
-CMD ["./opa", "run", "-s"]
+COPY --chown=stackable:stackable opa/stackable/bin /stackable/opa/bin
+
+ENV PATH="${PATH}:/stackable/opa:/stackable/opa/bin"
+
+CMD ["opa", "run", "-s"]

opa/stackable/bin/process-logs

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# This file was created by the Stackable developers.
+#
+# Processes incoming log messages. Log messages are filtered by the set log levels
+# and forwarded to the output destinations console and/or file.
+#
+
+set -euo pipefail
+
+if [ -z "${CONSOLE_LEVEL}" ]; then
+  echo "ERROR: env variable CONSOLE_LEVEL cannot be empty"
+  exit 1
+fi
+
+if [ -z "${FILE_LEVEL}" ]; then
+  echo "ERROR: env variable FILE_LEVEL cannot be empty"
+  exit 1
+fi
+
+if [ -z "${DECISION_LEVEL}" ]; then
+  echo "ERROR: env variable DECISION_LEVEL cannot be empty"
+  exit 1
+fi
+
+if [ -z "${SERVER_LEVEL}" ]; then
+  echo "ERROR: env variable SERVER_LEVEL cannot be empty"
+  exit 1
+fi
+
+if [ -z "${OPA_ROLLING_LOG_FILE_SIZE_BYTES}" ]; then
+  echo "ERROR: env variable OPA_ROLLING_LOG_FILE_SIZE_BYTES cannot be empty"
+  exit 1
+fi
+
+if [ -z "${OPA_ROLLING_LOG_FILES}" ]; then
+  echo "ERROR: env variable OPA_ROLLING_LOG_FILES cannot be empty"
+  exit 1
+fi
+
+if [ -z "${STACKABLE_LOG_DIR}" ]; then
+  echo "ERROR: env variable STACKABLE_LOG_DIR cannot be empty"
+  exit 1
+fi
+
+if [ -z "${CONTAINER_NAME}" ]; then
+  echo "ERROR: env variable CONTAINER_NAME cannot be empty"
+  exit 1
+fi
+
+get_levels() {
+    case $1 in
+        fatal)
+            echo '["fatal"]' ;;
+        error)
+            echo '["error","fatal"]' ;;
+        warn)
+            echo '["warn","error","fatal"]' ;;
+        info)
+            echo '["info","warn","error","fatal"]' ;;
+        debug)
+            echo '["debug","info","warn","error","fatal"]' ;;
+        trace)
+            echo '["trace","debug","info","warn","error","fatal"]' ;;
+        *)
+            echo '[]' ;;
+    esac
+}
+
+main() {
+    local DECISION_LEVELS
+    DECISION_LEVELS=$(get_levels "$DECISION_LEVEL")
+    local SERVER_LEVELS
+    SERVER_LEVELS=$(get_levels "$SERVER_LEVEL")
+    local CONSOLE_LEVELS
+    CONSOLE_LEVELS=$(get_levels "$CONSOLE_LEVEL")
+    local FILE_LEVELS
+    FILE_LEVELS=$(get_levels "$FILE_LEVEL")
+
+    jq -R -c --unbuffered --arg decision_levels "$DECISION_LEVELS" --arg server_levels "$SERVER_LEVELS" \
+    'try (fromjson | if .decision_id then .logger = "decision" else .logger = "server" end | select(((.logger == "decision") and (.level | inside($decision_levels))) or 
+        ((.logger == "server") and (.level | inside($server_levels))))) catch {"time":(now | todate),"level":"info","msg":"Could not process log message","error":true}' |
+    tee >(jq -c --unbuffered --arg file_levels "$FILE_LEVELS" 'select(.level | inside($file_levels))' \
+        > >(/stackable/multilog s"$OPA_ROLLING_LOG_FILE_SIZE_BYTES" n"$OPA_ROLLING_LOG_FILES" "$STACKABLE_LOG_DIR"/"$CONTAINER_NAME")) |
+    jq -c --unbuffered --arg console_levels "$CONSOLE_LEVELS" 'select(.level | inside($console_levels))'
+}
+
+main