Add function-mmsys-test-scene-detection

nagare-media · Feb 2, 2024 · 3f84d72 · 3f84d72
1 parent ab4372c
commit 3f84d72
Show file tree

Hide file tree

Showing 18 changed files with 1,281 additions and 8 deletions.
diff --git a/.github/workflows/container-images.yml b/.github/workflows/container-images.yml
@@ -20,6 +20,7 @@ jobs:
       matrix:
         image:
           - function-mmsys-test-encode
+          - function-mmsys-test-scene-detection
           - function-noop
           - function-sleep
           - gateway-nbmp

diff --git a/Makefile b/Makefile
@@ -202,14 +202,15 @@ clean: ## Cleanup build output
 .PHONY: image
 image: $(addprefix image-, $(IMAGES)) ## Build all container images
 
-## image-function-mmsys-test-encode:     ## Build mmsys-test-encode function container image
-## image-function-noop:                  ## Build noop function container image
-## image-function-sleep:                 ## Build sleep function container image
-## image-gateway-nbmp:                   ## Build NBMP gateway container image
-## image-workflow-manager:               ## Build workflow manager container image
-## image-workflow-manager-helper:        ## Build workflow manager helper container image
-## image-workflow-opentelemetry-adapter: ## Build workflow OpenTelemetry adapter container image
-## image-workflow-vacuum:                ## Build workflow vacuum container image
+## image-function-mmsys-test-encode:          ## Build mmsys-test-encode function container image
+## image-function-mmsys-test-scene-detection: ## Build mmsys-test-scene-detection function container image
+## image-function-noop:                       ## Build noop function container image
+## image-function-sleep:                      ## Build sleep function container image
+## image-gateway-nbmp:                        ## Build NBMP gateway container image
+## image-workflow-manager:                    ## Build workflow manager container image
+## image-workflow-manager-helper:             ## Build workflow manager helper container image
+## image-workflow-opentelemetry-adapter:      ## Build workflow OpenTelemetry adapter container image
+## image-workflow-vacuum:                     ## Build workflow vacuum container image
 image-%:
 	@	IMAGE="$*" \
 		IMAGE_REGISTRY="$(IMAGE_REGISTRY)" \

diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.generic.generic b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.generic.generic
@@ -0,0 +1,105 @@
+# syntax=docker/dockerfile:1.4
+
+ARG GOVERSION
+FROM --platform=${BUILDPLATFORM} docker.io/golang:${GOVERSION} AS build
+
+ARG SKAFFOLD_GO_GCFLAGS=""
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    go mod download
+COPY . .
+
+# BUILD_DATE changes ofter so move this section down to allow for caching
+ARG TARGETOS \
+    TARGETARCH \
+    VERSION \
+    GIT_COMMIT \
+    GIT_TREE_STATE \
+    BUILD_DATE
+
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/app/tmp,id=app-tmp-${TARGETARCH}-${TARGETVARIANT},sharing=locked \
+    make build-task-shim \
+      "OS=${TARGETOS}" \
+      "ARCH=${TARGETARCH}" \
+      "VERSION=${VERSION}" \
+      "GIT_COMMIT=${GIT_COMMIT}" \
+      "GIT_TREE_STATE=${GIT_TREE_STATE}" \
+      "BUILD_DATE=${BUILD_DATE}" \
+ && make build-functions \
+      "OS=${TARGETOS}" \
+      "ARCH=${TARGETARCH}" \
+      "VERSION=${VERSION}" \
+      "GIT_COMMIT=${GIT_COMMIT}" \
+      "GIT_TREE_STATE=${GIT_TREE_STATE}" \
+      "BUILD_DATE=${BUILD_DATE}"
+
+FROM docker.io/ubuntu:23.04
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+      ca-certificates \
+      ffmpeg \
+      libavcodec-extra \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV GOTRACEBACK=all
+
+ARG TARGETOS \
+    TARGETARCH \
+    VERSION \
+    GIT_COMMIT \
+    BUILD_DATE
+
+LABEL maintainer="Matthias Neugebauer <mtneug@mailbox.org>" \
+      org.opencontainers.image.created="${BUILD_DATE}" \
+      org.opencontainers.image.authors="Matthias Neugebauer <mtneug@mailbox.org>" \
+      org.opencontainers.image.url="ghcr.io/nagare-media/engine/function-mmsys-test-scene-detection" \
+      org.opencontainers.image.documentation="https://nagare.media" \
+      org.opencontainers.image.source="https://github.com/nagare-media/engine" \
+      org.opencontainers.image.version="${VERSION}" \
+      org.opencontainers.image.revision="${GIT_COMMIT}" \
+      org.opencontainers.image.vendor="nagare media" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.title="nagare media engine" \
+      org.opencontainers.image.description="nagare media engine — an open source modern cloud- and edge-native media workflow system running on Kubernetes" \
+      org.opencontainers.image.base.name="docker.io/ubuntu:23.04"
+
+WORKDIR /
+COPY --from=build "/app/bin/task-shim-${VERSION}-${TARGETOS}-${TARGETARCH}" /task-shim
+COPY --from=build "/app/bin/functions-${VERSION}-${TARGETOS}-${TARGETARCH}" /mmsys-test-scene-detection
+USER 65532:65532
+
+COPY <<EOF /config/task-shim.yaml
+apiVersion: engine.nagare.media/v1alpha1
+kind: TaskShimConfiguration
+task:
+  actions:
+    - name: execute function
+      action: task-shim.engine.nagare.media/exec
+      config:
+        command: /mmsys-test-scene-detection
+        args: ["/tmp/nbmp.tdd"]
+  onCreate:
+    - name: write task description document
+      action: task-shim.engine.nagare.media/file
+      config: |
+        path: /tmp/nbmp.tdd
+        content: |
+          {{ toJson .Task }}
+    - action: task-shim.engine.nagare.media/meta
+      config: start-task
+EOF
+
+# NBMP Task API
+EXPOSE 8888
+
+# event API
+EXPOSE 8080
+
+ENTRYPOINT [ "/task-shim" ]
+CMD [ "--config", "/config/task-shim.yaml" ]
diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.amd64 b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.amd64
@@ -0,0 +1 @@
+Dockerfile.generic.generic
diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.arm b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.arm
@@ -0,0 +1 @@
+Dockerfile.generic.generic
diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.arm64 b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.arm64
@@ -0,0 +1 @@
+Dockerfile.generic.generic
diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.ppc64le b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.ppc64le
@@ -0,0 +1 @@
+Dockerfile.generic.generic
diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.s390x b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.s390x
@@ -0,0 +1 @@
+Dockerfile.generic.generic
diff --git a/cmd/functions/cli/cli.go b/cmd/functions/cli/cli.go
@@ -36,6 +36,7 @@ import (
 
 	// Import nagare media functions to be included in this multi-binary.
 	_ "github.com/nagare-media/engine/internal/functions/mmsys-test-encode"
+	_ "github.com/nagare-media/engine/internal/functions/mmsys-test-scene-detection"
 	_ "github.com/nagare-media/engine/internal/functions/noop"
 	_ "github.com/nagare-media/engine/internal/functions/sleep"
 )

diff --git a/config/samples/k8s/mmsys-test-scene-detection/README.md b/config/samples/k8s/mmsys-test-scene-detection/README.md
@@ -0,0 +1,114 @@
+# Example for `mmsys-test-scene-detection` function
+
+This example shows task execution of the `mmsys-test-scene-detection` function demonstrating MPEG NBMP task error recovery using event sourcing. The NBMP tasks are created directly without going through the NBMP Workflow API and the full workflow manager. However, the nagare media engine `workflow-manager-helper` is deployed as a sidecar container next to the task container. The `workflow-manager-helper` will read the mounted secret and configure the task using the NBMP Task API. It then monitors the execution status.
+
+The `mmsys-test-scene-detection` function can be configured with the following options:
+
+| Config                                                                           | Description                                                         |
+| -------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
+| `mmsys-test-scene-detection.engine.nagare.media/test`                            | Execution test/mode to use (s. below). (required)                   |
+| `mmsys-test-scene-detection.engine.nagare.media/max-number-of-simulated-crashes` | Number crashes, i.e. hard terminations, to simulate (defaults to 1) |
+| `mmsys-test-scene-detection.engine.nagare.media/simulated-crash-wait-duration`   | Time to wait until simulated crash is triggered. (defaults to 20s) |
+
+The following execution tests/modes exist:
+
+| Test             | Description                                                                   |
+| ---------------- | ----------------------------------------------------------------------------- |
+| baseline         | Simple scene detection.                                                       |
+| test-no-recovery | Scene detection with simulated crashes and no recovery.                       |
+| test-recovery    | Scene detection with simulated crashes and recovery from last detected scene. |
+
+## Host environment
+
+The following sections assume that certain software is installed on the host environment:
+
+* Docker
+* `sh` (UNIX shell)
+* `bash`
+* `make`
+* `kind`
+* `kubectl`
+* `helm`
+
+Alternatively, the commands can be executed in a Docker container that serves as the test environment. Use the following commands to initialize a new container and install the necessary tools. Then follow the guide normally but execute the commands within the container environment.
+
+```sh
+# create kind network; ignore if it already exist
+$ docker network create \
+    --driver bridge \
+    --ipv6 --subnet fc00:f853:ccd:e793::/64 \
+    --opt com.docker.network.bridge.enable_ip_masquerade=true \
+    --opt com.docker.network.driver.mtu=65535 \
+    kind
+
+$ docker run --rm -it --privileged \
+    --name nagare-media-engine-test-env \
+    --network kind \
+    -v "/var/run/docker.sock:/var/run/docker.sock" \
+    -v "$PWD:/src" \
+    -w /src \
+    docker.io/library/alpine:edge
+
+# within the container:
+
+# install necessary tools
+docker$ echo "https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
+docker$ apk add \
+          bash \
+          docker-cli \
+          helm \
+          kind \
+          kubectl \
+          make
+
+# continue with the guide
+docker$ ...
+
+# exit (with automatic cleanup)
+docker$ exit
+```
+
+## Create Kubernetes Cluster
+
+```sh
+$ make kind-up
+$ kubectl create namespace mmsys-test-scene-detection
+```
+
+## Upload Test Media to In-Cluster S3
+
+```sh
+$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-upload-media.yaml
+# wait until job is complete
+$ kubectl logs -f -n mmsys-test-scene-detection jobs/upload-media
+```
+
+## Create Tasks (`Secret` + `Job`)
+
+In every test case, the output can be accessed at <http://s3.local.gd/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt> (`s3.local.gd` resolves to 127.0.0.1 assuming the local DNS resolver allows that; the Kubernetes cluster has an Ingress that routes `s3.local.gd` requests to the in-cluster S3).
+
+```sh
+# baseline
+$ kubectl -n mmsys-test-scene-detection create secret generic workflow-manager-helper-data-mmsys-test-scene-detection-baseline \
+    --from-file=data.yaml=config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-baseline.yaml
+$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml
+$ kubectl logs -f -n mmsys-test-scene-detection jobs/task-mmsys-test-scene-detection-baseline -c function
+
+# test-no-recovery
+$ kubectl -n mmsys-test-scene-detection create secret generic workflow-manager-helper-data-mmsys-test-scene-detection-test-no-recovery \
+    --from-file=data.yaml=config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-no-recovery.yaml
+$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-no-recovery.yaml
+$ kubectl logs -f -n mmsys-test-scene-detection jobs/task-mmsys-test-scene-detection-test-no-recovery -c function
+
+# test-recovery
+$ kubectl -n mmsys-test-scene-detection create secret generic workflow-manager-helper-data-mmsys-test-scene-detection-test-recovery \
+    --from-file=data.yaml=config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-recovery.yaml
+$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-recovery.yaml
+$ kubectl logs -f -n mmsys-test-scene-detection jobs/task-mmsys-test-scene-detection-test-recovery -c function
+```
+
+## Delete Kubernetes Cluster
+
+```sh
+$ make kind-down
+```
diff --git a/.../samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml b/.../samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml
@@ -0,0 +1,89 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: task-mmsys-test-scene-detection-baseline
+spec:
+  backoffLimit: 10
+  podFailurePolicy:
+    rules:
+      - action: Ignore
+        onPodConditions:
+          - type: DisruptionTarget
+  ttlSecondsAfterFinished: 86400
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: function
+    spec:
+      restartPolicy: Never # recreate pod by job controller
+      terminationGracePeriodSeconds: 35
+      securityContext:
+        runAsUser: 0
+        runAsGroup: 0
+      containers:
+        - name: function
+          imagePullPolicy: Always
+          image: ghcr.io/nagare-media/engine/function-mmsys-test-scene-detection:dev
+          ports:
+            - name: task-api
+              containerPort: 8888
+              protocol: TCP
+          startupProbe:
+            failureThreshold: 60
+            periodSeconds: 2
+            httpGet:
+              path: /healthz
+              port: task-api
+          readinessProbe:
+            periodSeconds: 2
+            httpGet:
+              path: /readyz
+              port: task-api
+          livenessProbe:
+            periodSeconds: 2
+            httpGet:
+              path: /healthz
+              port: task-api
+          resources:
+            requests:
+              cpu: 7000m
+              memory: 2Gi
+        - name: workflow-manager-helper
+          imagePullPolicy: Always
+          image: ghcr.io/nagare-media/engine/workflow-manager-helper:dev
+          args: [ "--config", "/config/workflow-manager-helper.yaml", "/run/secrets/engine.nagare.media/task/data.yaml" ]
+          ports:
+            - name: event-api
+              containerPort: 8181
+              protocol: TCP
+          startupProbe:
+            failureThreshold: 60
+            periodSeconds: 2
+            httpGet:
+              path: /healthz
+              port: event-api
+          readinessProbe:
+            periodSeconds: 2
+            httpGet:
+              path: /readyz
+              port: event-api
+          livenessProbe:
+            periodSeconds: 2
+            httpGet:
+              path: /healthz
+              port: event-api
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+          volumeMounts:
+            - name: task-data
+              mountPath: /run/secrets/engine.nagare.media/task
+              readOnly: true
+      volumes:
+        - name: task-data
+          secret:
+            secretName: workflow-manager-helper-data-mmsys-test-scene-detection-baseline