feat: Add log processing script to opa for decision logging

CHANGELOG.md

@@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file.
 - stackable-base: Mitigate CVE-2023-37920 by removing e-Tugra root certificates ([#673]).
 - hdfs: Exclude unused jars and mitigate snappy-java CVEs by bumping dependency ([#682]).
 - druid: Build from source ([#684]).
+- opa: Add log processing script to opa for decision logging ([#695]).
 
 ### Changed
 
@@ -87,6 +88,7 @@ All notable changes to this project will be documented in this file.
 [#684]: https://github.com/stackabletech/docker-images/pull/684
 [#685]: https://github.com/stackabletech/docker-images/pull/685
 [#688]: https://github.com/stackabletech/docker-images/pull/688
+[#695]: https://github.com/stackabletech/docker-images/pull/695
 
 ## [24.3.0] - 2024-03-20
 

opa/Dockerfile

@@ -109,6 +109,9 @@ LABEL name="Open Policy Agent" \
       description="This image is deployed by the Stackable Operator for OPA."
 
 RUN microdnf update && \
+    microdnf install \
+    # Required for filtering logs
+    jq && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
@@ -121,4 +124,8 @@ COPY --from=opa-builder /opa/opa /stackable/opa/opa
 COPY --from=opa-bundle-builder --chown=stackable:stackable /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder
 COPY --from=multilog-builder --chown=stackable:stackable /daemontools/admin/daemontools/command/multilog /stackable/multilog
 
-CMD ["./opa", "run", "-s"]
+COPY --chown=stackable:stackable opa/stackable/bin /stackable/opa/bin
+
+ENV PATH="${PATH}:/stackable/opa:/stackable/opa/bin"
+
+CMD ["opa", "run", "-s"]

opa/stackable/bin/process-logs

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+#
+# This file was created by the Stackable developers.
+#
+# Usage: process-logs <options>
+# Options:
+#   --console-log-level <log-level>
+#   --file-log-level <log-level>
+#   --decision-log-level <log-level>
+#   --server-log-level <log-level>
+#   --opa-rolling-log-file-size-bytes <maximum-file-size>
+#   --opa-rolling-log-files <number-of-log-files>
+#   --stackable-log-dir <path-to-log-dir>
+#   --container-name <container-name>
+#
+# Processes incoming log messages. Log messages are filtered by the set log levels
+# and forwarded to the output destinations console and/or file.
+#
+
+set -eo pipefail
+
+parse_args() {
+  while true; do
+    case $1 in
+        --console-log-level)
+            shift
+            CONSOLE_LEVEL="$1"
+            ;;
+        --file-log-level)
+            shift
+            FILE_LEVEL="$1"
+            ;;
+        --decision-log-level)
+            shift
+            DECISION_LEVEL="$1"
+            ;;
+        --server-log-level)
+            shift
+            SERVER_LEVEL="$1"
+            ;;
+        --opa-rolling-log-file-size-bytes)
+            shift
+            OPA_ROLLING_LOG_FILE_SIZE_BYTES="$1"
+            ;;
+        --opa-rolling-log-files)
+            shift
+            OPA_ROLLING_LOG_FILES="$1"
+            ;;
+        --stackable-log-dir)
+            shift
+            STACKABLE_LOG_DIR="$1"
+            ;;
+        --container-name)
+            shift
+            CONTAINER_NAME="$1"
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+  done
+
+  if [ -z "${CONSOLE_LEVEL}" ]; then
+    echo "ERROR: parameter --console-log-level cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${FILE_LEVEL}" ]; then
+    echo "ERROR: parameter --file-log-level cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${DECISION_LEVEL}" ]; then
+    echo "ERROR: parameter --decision-log-level cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${SERVER_LEVEL}" ]; then
+    echo "ERROR: parameter --server-log-level cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${OPA_ROLLING_LOG_FILE_SIZE_BYTES}" ]; then
+    echo "ERROR: parameter --opa-rolling-log-file-size-bytes cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${OPA_ROLLING_LOG_FILES}" ]; then
+    echo "ERROR: parameter --opa-rolling-log-files cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${STACKABLE_LOG_DIR}" ]; then
+    echo "ERROR: parameter --stackable-log-dir cannot be empty"
+    exit 1
+  fi
+
+  if [ -z "${CONTAINER_NAME}" ]; then
+    echo "ERROR: parameter --container-name cannot be empty"
+    exit 1
+  fi
+}
+
+get_levels() {
+    case $1 in
+        fatal)
+            echo '["fatal"]' ;;
+        error)
+            echo '["error","fatal"]' ;;
+        warn)
+            echo '["warn","error","fatal"]' ;;
+        info)
+            echo '["info","warn","error","fatal"]' ;;
+        debug)
+            echo '["debug","info","warn","error","fatal"]' ;;
+        trace)
+            echo '["trace","debug","info","warn","error","fatal"]' ;;
+        *)
+            echo '[""]' ;;
+    esac
+}
+
+main() {
+    parse_args "$@"
+
+    local DECISION_LEVELS
+    DECISION_LEVELS=$(get_levels "$DECISION_LEVEL")
+    local SERVER_LEVELS
+    SERVER_LEVELS=$(get_levels "$SERVER_LEVEL")
+    local CONSOLE_LEVELS
+    CONSOLE_LEVELS=$(get_levels "$CONSOLE_LEVEL")
+    local FILE_LEVELS
+    FILE_LEVELS=$(get_levels "$FILE_LEVEL")
+
+    jq -R -c --unbuffered --arg decision_levels "$DECISION_LEVELS" --arg server_levels "$SERVER_LEVELS" \
+    'try (fromjson | if .decision_id then .logger = "decision" else .logger = "server" end | select(((.logger == "decision") and (.level | inside($decision_levels))) or 
+        ((.logger == "server") and (.level | inside($server_levels))))) catch {"level":"warn","msg":"Could not process log message"}' |
+    tee >(jq -c --unbuffered --arg file_levels "$FILE_LEVELS" 'select(.level | inside($file_levels))' \
+        > >(/stackable/multilog s"$OPA_ROLLING_LOG_FILE_SIZE_BYTES" n"$OPA_ROLLING_LOG_FILES" "$STACKABLE_LOG_DIR"/"$CONTAINER_NAME")) |
+    jq -c --unbuffered --arg console_levels "$CONSOLE_LEVELS" 'select(.level | inside($console_levels))'
+}
+
+main "$@"