Initial commit

.devcontainer/devcontainer.json

@@ -0,0 +1,36 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu
+{
+	"name": "Ubuntu",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/base:bullseye",
+	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/devcontainers/features/python:1": {},
+		"ghcr.io/edouard-lopez/devcontainer-features/bats:0": {}
+	},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "bash .devcontainer/post-create-script.sh"
+	// Configure tool-specific properties.
+	,
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"jetmartin.bats@0.1.9",
+				"timonwong.shellcheck@0.29.4"
+			],
+			"settings": {
+				"files.eol": "\n",
+				"editor.tabSize": 4,
+				"editor.formatOnPaste": false,
+				"editor.formatOnSave": true,
+				"editor.formatOnType": true,
+				"files.trimTrailingWhitespace": true
+			}
+		}
+	}
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/post-create-script.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Install python dependencies
+pip3 install --user -r requirements_dev.txt
+
+# Install bats helpers
+[ -d tests/bats-helpers ] && rm -rf tests/bats-helpers && mkdir -p tests/bats-helpers
+
+git clone --depth 1 https://github.com/bats-core/bats-support.git tests/bats-helpers/bats-support || true
+git clone --depth 1 https://github.com/bats-core/bats-assert.git tests/bats-helpers/bats-assert || true
+git clone --depth 1 https://github.com/bats-core/bats-file.git tests/bats-helpers/bats-file || true

.github/workflows/ci-checks.yml

@@ -0,0 +1,50 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CI checks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  linting-bash:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v1
+      - name: Run ShellCheck
+        uses: azohra/shell-linter@latest
+
+
+  linting-python:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_dev.txt
+    - name: Run black
+      run: |
+        black --check bin/*
+    - name: Run pylint
+      run: |
+        pylint bin/*
+
+  bats:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v3
+      - name: Build and run Dev Container task
+        uses: devcontainers/ci@v0.3
+        with:
+          runCmd: |
+            bats tests/

.github/workflows/release.yml

@@ -0,0 +1,36 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          push: true
+          tags: |
+            ghcr.io/mo-rise/porla:latest
+            ghcr.io/mo-rise/porla:${{ github.event.release.tag_name }}
+      -
+        name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11-slim-bullseye
+
+RUN apt-get update && apt-get install -y \
+    netcat-openbsd \
+    socat \
+    mosquitto-clients \
+    jq \
+    jo \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY --chmod=555 ./bin/* /usr/local/bin/
+COPY bash-init.sh /bash-init.sh
+
+ENV BASH_ENV=/bash-init.sh
+
+ENTRYPOINT ["/bin/bash", "-l", "-c"]

README.md

@@ -1,2 +1,35 @@
 # porla
-Pipeline-Oriented Real-time Logging Assistant
+**P**ipeline-**O**riented **R**eal-time **L**ogging **A**ssistant or simply a basic toolbox for handling of line-based streaming data using linux pipes and some handy command-line utilities
+
+    "porla" is also a Swedish word for the soothing sound of running water from a small stream stream, typically set in an enchanted forest.
+
+## Motivation, purpose and mental map
+
+
+![schematic](./porla.svg)
+
+## Usage
+
+It is packaged as a docker image, available here: https://github.com/orgs/MO-RISE/packages
+
+The image expects a "run" command inputted on startup. Using docker run, this would manifest as such:
+```
+docker run --network=host porla "<command>"
+```
+
+Using docker-compose it would look like this:
+```
+version: '3'
+services:
+    service_1:
+        image: ghcr.io/mo-rise/porla
+        network_mode: host
+        restart: always
+        command: ["<command>"]
+```
+
+### Built-in functionality
+
+TODO
+
+### Examples

bash-init.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function to_bus () {
+    socat STDIN "UDP4-DATAGRAM:239.111.42.$1:12737"
+}
+export to_bus
+
+function from_bus () {
+    socat "UDP4-RECV:12737,reuseaddr,ip-add-membership=239.111.42.$1:0.0.0.0" STDOUT
+}
+export from_bus
+
+function record () {
+    tee -a "$1" > /dev/null
+}
+export record
+
+function branch () {
+    tee >("$@")
+}
+export branch

bin/b64

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is base64 encoded with no wrapping and ended with a newline.
+"""
+
+import sys
+import argparse
+from base64 import b64encode, b64decode
+
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument("--encode", action="store_true", default=False)
+group.add_argument("--decode", action="store_true", default=False)
+
+args = parser.parse_args()
+
+if args.encode:
+    for line in sys.stdin:
+        sys.stdout.write(b64encode(line.encode()).decode() + "\n")
+        sys.stdout.flush()
+
+else:
+    for line in sys.stdin:
+        sys.stdout.write(b64decode(line.encode()).decode())
+        sys.stdout.flush()

bin/jsonify

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is parsed according to the specification provided by the user
+and assembled into a json object.
+"""
+
+import sys
+import json
+import logging
+import warnings
+import argparse
+
+import parse
+
+# Parse cli arguments
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--log-level", type=lambda level: getattr(logging, level), default=logging.WARNING
+)
+parser.add_argument(
+    "specification",
+    type=str,
+    help="Example: '{timestamp} {data}',"
+    "See https://github.com/r1chardj0n3s/parse#format-specification",
+)
+
+args = parser.parse_args()
+
+# Setup logger
+logging.basicConfig(
+    format="%(asctime)s %(levelname)s %(name)s %(message)s", level=args.log_level
+)
+logging.captureWarnings(True)
+warnings.filterwarnings("once")
+
+logger = logging.getLogger("jsonify")
+
+
+# Compile pattern
+pattern = parse.compile(args.specification)
+
+# Start processing
+for line in sys.stdin:
+    logger.debug(line)
+    res = pattern.parse(line.rstrip())
+
+    if not res:
+        logger.error(
+            "Could not parse line: %s according to the specification: %s",
+            line,
+            args.specification,
+        )
+        continue
+
+    sys.stdout.write(json.dumps(res.named) + "\n")
+    sys.stdout.flush()

bin/timestamp

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+"""
+Command line utility tool for processing input from stdin. Each line on the
+input stream is prepended with the current timestamp in the format requested
+by the user.
+"""
+
+# pylint: disable=unnecessary-lambda-assignment
+
+import sys
+import time
+import argparse
+from datetime import datetime, timezone
+
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument("--epoch", action="store_true", default=False)
+group.add_argument("--rfc3339", action="store_true", default=False)
+
+args = parser.parse_args()
+
+if args.epoch:
+    stamp = lambda: f"{time.time():.6f}"
+
+elif args.rfc3339:
+    stamp = lambda: datetime.now(timezone.utc).isoformat()
+
+
+for line in sys.stdin:
+    sys.stdout.write(f"{stamp()} {line}")
+    sys.stdout.flush()