Initial Commit

.github/workflows/image.yaml

@@ -0,0 +1,17 @@
+name: Build image
+
+on:
+ push:
+ branches: ["main"]
+ tags:
+ - 'v*.*.*' # glob for semver tags (including prereleases)
+ pull_request:
+ branches: [main]
+
+jobs:
+ call-build-image:
+ uses: UW-Macrostrat/build-push/build-push.yaml@main
+ with:
+ image: 'hub.opensciencegrid.org/macrostrat/traefik-log-exporter'
+ username: ${{ vars.HARBOR_CLI_NAME }}
+ password: ${{ secrets.HARBOR_CLI_SECRET }}

.gitignore

@@ -0,0 +1,2 @@
+test-logs
+.env

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "import-script"]
+ path = import-script
+ url = git@github.com:matomo-org/matomo-log-analytics.git

Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.8
+
+# Install cron
+RUN apt-get update && apt-get install -y cron
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the necessary files
+COPY ./import-script/import_logs.py import_logs.py
+COPY ./import.sh import.sh
+COPY ./logrotate.sh logrotate.sh
+
+# Make the scripts executable
+RUN chmod +x import.sh logrotate.sh
+
+# Give execution rights on the cron job
+RUN chmod 0644 /etc/cron.d/import-cron
+
+# Apply cron job
+RUN crontab /etc/cron.d/import-cron
+
+# Create the log file to be able to run tail
+RUN touch /var/log/cron.log
+
+# Start the cron service and the application
+CMD cron && tail -f /var/log/cron.log

README.md

@@ -1 +1,14 @@
-# matomo-log-scraper
+# matomo-log-scraper
+
+This service runs as a sidecar to the Traefik pod. It rotates the logs daily and sends them to the Matomo server for analysis. 
+
+It keeps 30 days of logs compressed in the PVC on the cluster for debugging purposes.
+
+# Testing
+
+```bash
+source .env
+
+# Construct the command with the environment variables
+python3 import-script/import_logs.py "$LOG_PATH" --dry-run --url "$MATOMO_URL" --token-auth "$API_TOKEN"
+```

crontab

@@ -0,0 +1,2 @@
+# Run logrotate.sh daily at midnight
+0 0 * * * /app/logrotate.sh >> /var/log/cron.log 2>&1

example.env

@@ -0,0 +1,3 @@
+LOG_PATH=./test-logs/traefik.json
+API_TOKEN=
+MATOMO_URL=https://analytics.macrostrat.org

import.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Define the log file and the rotated log file
+ROTATED_LOG_FILE="$LOG_PATH.1"
+
+# Check if the rotated log file exists
+if [ -f "$ROTATED_LOG_FILE" ]; then
+ echo "Log rotation successful. Processing the rotated log file."
+ # Call the import script
+ python3 /app/import_logs.py "$ROTATED_LOG_FILE" --url "$MATOMO_URL" --token-auth "$API_TOKEN"
+else
+ echo "Rotated log file not found."
+ exit 1
+fi

logrotate.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# Create a temporary logrotate configuration file
+cat <<EOF > /tmp/logrotate.conf
+$LOG_PATH {
+ daily
+ rotate 31
+ compress
+ delaycompress
+ postrotate
+ /app/import.sh
+ endscript
+}
+EOF
+
+# Run logrotate with the temporary configuration file
+/usr/sbin/logrotate /tmp/logrotate.conf
+
+# Clean up the temporary configuration file
+rm /tmp/logrotate.conf