From 3f84d720a0fa74f86e8fb6266362b71641fad596 Mon Sep 17 00:00:00 2001 From: Matthias Neugebauer Date: Fri, 2 Feb 2024 07:04:48 +0100 Subject: [PATCH] Add function-mmsys-test-scene-detection --- .github/workflows/container-images.yml | 1 + Makefile | 17 +- .../Dockerfile.generic.generic | 105 +++ .../Dockerfile.linux.amd64 | 1 + .../Dockerfile.linux.arm | 1 + .../Dockerfile.linux.arm64 | 1 + .../Dockerfile.linux.ppc64le | 1 + .../Dockerfile.linux.s390x | 1 + cmd/functions/cli/cli.go | 1 + .../k8s/mmsys-test-scene-detection/README.md | 114 ++++ ...k-mmsys-test-scene-detection-baseline.yaml | 89 +++ ...test-scene-detection-test-no-recovery.yaml | 89 +++ ...ys-test-scene-detection-test-recovery.yaml | 89 +++ .../job-upload-media.yaml | 21 + ...a_mmsys-test-scene-detection-baseline.yaml | 40 ++ ...test-scene-detection-test-no-recovery.yaml | 50 ++ ...ys-test-scene-detection-test-recovery.yaml | 50 ++ .../mmsys-test-scene-detection/func.go | 618 ++++++++++++++++++ 18 files changed, 1281 insertions(+), 8 deletions(-) create mode 100644 build/package/image/function-mmsys-test-scene-detection/Dockerfile.generic.generic create mode 120000 build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.amd64 create mode 120000 build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.arm create mode 120000 build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.arm64 create mode 120000 build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.ppc64le create mode 120000 build/package/image/function-mmsys-test-scene-detection/Dockerfile.linux.s390x create mode 100644 config/samples/k8s/mmsys-test-scene-detection/README.md create mode 100644 config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml create mode 100644 config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-no-recovery.yaml create mode 100644 config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-recovery.yaml create mode 100644 config/samples/k8s/mmsys-test-scene-detection/job-upload-media.yaml create mode 100644 config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-baseline.yaml create mode 100644 config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-no-recovery.yaml create mode 100644 config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-recovery.yaml create mode 100644 internal/functions/mmsys-test-scene-detection/func.go diff --git a/.github/workflows/container-images.yml b/.github/workflows/container-images.yml index e1118b8..4f6eaf1 100644 --- a/.github/workflows/container-images.yml +++ b/.github/workflows/container-images.yml @@ -20,6 +20,7 @@ jobs: matrix: image: - function-mmsys-test-encode + - function-mmsys-test-scene-detection - function-noop - function-sleep - gateway-nbmp diff --git a/Makefile b/Makefile index fe26d88..623a72c 100644 --- a/Makefile +++ b/Makefile @@ -202,14 +202,15 @@ clean: ## Cleanup build output .PHONY: image image: $(addprefix image-, $(IMAGES)) ## Build all container images -## image-function-mmsys-test-encode: ## Build mmsys-test-encode function container image -## image-function-noop: ## Build noop function container image -## image-function-sleep: ## Build sleep function container image -## image-gateway-nbmp: ## Build NBMP gateway container image -## image-workflow-manager: ## Build workflow manager container image -## image-workflow-manager-helper: ## Build workflow manager helper container image -## image-workflow-opentelemetry-adapter: ## Build workflow OpenTelemetry adapter container image -## image-workflow-vacuum: ## Build workflow vacuum container image +## image-function-mmsys-test-encode: ## Build mmsys-test-encode function container image +## image-function-mmsys-test-scene-detection: ## Build mmsys-test-scene-detection function container image +## image-function-noop: ## Build noop function container image +## image-function-sleep: ## Build sleep function container image +## image-gateway-nbmp: ## Build NBMP gateway container image +## image-workflow-manager: ## Build workflow manager container image +## image-workflow-manager-helper: ## Build workflow manager helper container image +## image-workflow-opentelemetry-adapter: ## Build workflow OpenTelemetry adapter container image +## image-workflow-vacuum: ## Build workflow vacuum container image image-%: @ IMAGE="$*" \ IMAGE_REGISTRY="$(IMAGE_REGISTRY)" \ diff --git a/build/package/image/function-mmsys-test-scene-detection/Dockerfile.generic.generic b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.generic.generic new file mode 100644 index 0000000..a5d76bc --- /dev/null +++ b/build/package/image/function-mmsys-test-scene-detection/Dockerfile.generic.generic @@ -0,0 +1,105 @@ +# syntax=docker/dockerfile:1.4 + +ARG GOVERSION +FROM --platform=${BUILDPLATFORM} docker.io/golang:${GOVERSION} AS build + +ARG SKAFFOLD_GO_GCFLAGS="" + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/root/.cache/go-build \ + go mod download +COPY . . + +# BUILD_DATE changes ofter so move this section down to allow for caching +ARG TARGETOS \ + TARGETARCH \ + VERSION \ + GIT_COMMIT \ + GIT_TREE_STATE \ + BUILD_DATE + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/app/tmp,id=app-tmp-${TARGETARCH}-${TARGETVARIANT},sharing=locked \ + make build-task-shim \ + "OS=${TARGETOS}" \ + "ARCH=${TARGETARCH}" \ + "VERSION=${VERSION}" \ + "GIT_COMMIT=${GIT_COMMIT}" \ + "GIT_TREE_STATE=${GIT_TREE_STATE}" \ + "BUILD_DATE=${BUILD_DATE}" \ + && make build-functions \ + "OS=${TARGETOS}" \ + "ARCH=${TARGETARCH}" \ + "VERSION=${VERSION}" \ + "GIT_COMMIT=${GIT_COMMIT}" \ + "GIT_TREE_STATE=${GIT_TREE_STATE}" \ + "BUILD_DATE=${BUILD_DATE}" + +FROM docker.io/ubuntu:23.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + ffmpeg \ + libavcodec-extra \ + && rm -rf /var/lib/apt/lists/* + +ENV GOTRACEBACK=all + +ARG TARGETOS \ + TARGETARCH \ + VERSION \ + GIT_COMMIT \ + BUILD_DATE + +LABEL maintainer="Matthias Neugebauer " \ + org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.authors="Matthias Neugebauer " \ + org.opencontainers.image.url="ghcr.io/nagare-media/engine/function-mmsys-test-scene-detection" \ + org.opencontainers.image.documentation="https://nagare.media" \ + org.opencontainers.image.source="https://github.com/nagare-media/engine" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${GIT_COMMIT}" \ + org.opencontainers.image.vendor="nagare media" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="nagare media engine" \ + org.opencontainers.image.description="nagare media engine — an open source modern cloud- and edge-native media workflow system running on Kubernetes" \ + org.opencontainers.image.base.name="docker.io/ubuntu:23.04" + +WORKDIR / +COPY --from=build "/app/bin/task-shim-${VERSION}-${TARGETOS}-${TARGETARCH}" /task-shim +COPY --from=build "/app/bin/functions-${VERSION}-${TARGETOS}-${TARGETARCH}" /mmsys-test-scene-detection +USER 65532:65532 + +COPY <> /etc/apk/repositories +docker$ apk add \ + bash \ + docker-cli \ + helm \ + kind \ + kubectl \ + make + +# continue with the guide +docker$ ... + +# exit (with automatic cleanup) +docker$ exit +``` + +## Create Kubernetes Cluster + +```sh +$ make kind-up +$ kubectl create namespace mmsys-test-scene-detection +``` + +## Upload Test Media to In-Cluster S3 + +```sh +$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-upload-media.yaml +# wait until job is complete +$ kubectl logs -f -n mmsys-test-scene-detection jobs/upload-media +``` + +## Create Tasks (`Secret` + `Job`) + +In every test case, the output can be accessed at (`s3.local.gd` resolves to 127.0.0.1 assuming the local DNS resolver allows that; the Kubernetes cluster has an Ingress that routes `s3.local.gd` requests to the in-cluster S3). + +```sh +# baseline +$ kubectl -n mmsys-test-scene-detection create secret generic workflow-manager-helper-data-mmsys-test-scene-detection-baseline \ + --from-file=data.yaml=config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-baseline.yaml +$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml +$ kubectl logs -f -n mmsys-test-scene-detection jobs/task-mmsys-test-scene-detection-baseline -c function + +# test-no-recovery +$ kubectl -n mmsys-test-scene-detection create secret generic workflow-manager-helper-data-mmsys-test-scene-detection-test-no-recovery \ + --from-file=data.yaml=config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-no-recovery.yaml +$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-no-recovery.yaml +$ kubectl logs -f -n mmsys-test-scene-detection jobs/task-mmsys-test-scene-detection-test-no-recovery -c function + +# test-recovery +$ kubectl -n mmsys-test-scene-detection create secret generic workflow-manager-helper-data-mmsys-test-scene-detection-test-recovery \ + --from-file=data.yaml=config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-recovery.yaml +$ kubectl -n mmsys-test-scene-detection apply -f config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-recovery.yaml +$ kubectl logs -f -n mmsys-test-scene-detection jobs/task-mmsys-test-scene-detection-test-recovery -c function +``` + +## Delete Kubernetes Cluster + +```sh +$ make kind-down +``` diff --git a/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml b/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml new file mode 100644 index 0000000..f44b067 --- /dev/null +++ b/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-baseline.yaml @@ -0,0 +1,89 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: task-mmsys-test-scene-detection-baseline +spec: + backoffLimit: 10 + podFailurePolicy: + rules: + - action: Ignore + onPodConditions: + - type: DisruptionTarget + ttlSecondsAfterFinished: 86400 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: function + spec: + restartPolicy: Never # recreate pod by job controller + terminationGracePeriodSeconds: 35 + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: function + imagePullPolicy: Always + image: ghcr.io/nagare-media/engine/function-mmsys-test-scene-detection:dev + ports: + - name: task-api + containerPort: 8888 + protocol: TCP + startupProbe: + failureThreshold: 60 + periodSeconds: 2 + httpGet: + path: /healthz + port: task-api + readinessProbe: + periodSeconds: 2 + httpGet: + path: /readyz + port: task-api + livenessProbe: + periodSeconds: 2 + httpGet: + path: /healthz + port: task-api + resources: + requests: + cpu: 7000m + memory: 2Gi + - name: workflow-manager-helper + imagePullPolicy: Always + image: ghcr.io/nagare-media/engine/workflow-manager-helper:dev + args: [ "--config", "/config/workflow-manager-helper.yaml", "/run/secrets/engine.nagare.media/task/data.yaml" ] + ports: + - name: event-api + containerPort: 8181 + protocol: TCP + startupProbe: + failureThreshold: 60 + periodSeconds: 2 + httpGet: + path: /healthz + port: event-api + readinessProbe: + periodSeconds: 2 + httpGet: + path: /readyz + port: event-api + livenessProbe: + periodSeconds: 2 + httpGet: + path: /healthz + port: event-api + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: task-data + mountPath: /run/secrets/engine.nagare.media/task + readOnly: true + volumes: + - name: task-data + secret: + secretName: workflow-manager-helper-data-mmsys-test-scene-detection-baseline diff --git a/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-no-recovery.yaml b/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-no-recovery.yaml new file mode 100644 index 0000000..79360fe --- /dev/null +++ b/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-no-recovery.yaml @@ -0,0 +1,89 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: task-mmsys-test-scene-detection-test-no-recovery +spec: + backoffLimit: 10 + podFailurePolicy: + rules: + - action: Ignore + onPodConditions: + - type: DisruptionTarget + ttlSecondsAfterFinished: 86400 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: function + spec: + restartPolicy: Never # recreate pod by job controller + terminationGracePeriodSeconds: 35 + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: function + imagePullPolicy: Always + image: ghcr.io/nagare-media/engine/function-mmsys-test-scene-detection:dev + ports: + - name: task-api + containerPort: 8888 + protocol: TCP + startupProbe: + failureThreshold: 60 + periodSeconds: 2 + httpGet: + path: /healthz + port: task-api + readinessProbe: + periodSeconds: 2 + httpGet: + path: /readyz + port: task-api + livenessProbe: + periodSeconds: 2 + httpGet: + path: /healthz + port: task-api + resources: + requests: + cpu: 7000m + memory: 2Gi + - name: workflow-manager-helper + imagePullPolicy: Always + image: ghcr.io/nagare-media/engine/workflow-manager-helper:dev + args: [ "--config", "/config/workflow-manager-helper.yaml", "/run/secrets/engine.nagare.media/task/data.yaml" ] + ports: + - name: event-api + containerPort: 8181 + protocol: TCP + startupProbe: + failureThreshold: 60 + periodSeconds: 2 + httpGet: + path: /healthz + port: event-api + readinessProbe: + periodSeconds: 2 + httpGet: + path: /readyz + port: event-api + livenessProbe: + periodSeconds: 2 + httpGet: + path: /healthz + port: event-api + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: task-data + mountPath: /run/secrets/engine.nagare.media/task + readOnly: true + volumes: + - name: task-data + secret: + secretName: workflow-manager-helper-data-mmsys-test-scene-detection-test-no-recovery diff --git a/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-recovery.yaml b/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-recovery.yaml new file mode 100644 index 0000000..5093edb --- /dev/null +++ b/config/samples/k8s/mmsys-test-scene-detection/job-task-mmsys-test-scene-detection-test-recovery.yaml @@ -0,0 +1,89 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: task-mmsys-test-scene-detection-test-recovery +spec: + backoffLimit: 10 + podFailurePolicy: + rules: + - action: Ignore + onPodConditions: + - type: DisruptionTarget + ttlSecondsAfterFinished: 86400 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: function + spec: + restartPolicy: Never # recreate pod by job controller + terminationGracePeriodSeconds: 35 + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: function + imagePullPolicy: Always + image: ghcr.io/nagare-media/engine/function-mmsys-test-scene-detection:dev + ports: + - name: task-api + containerPort: 8888 + protocol: TCP + startupProbe: + failureThreshold: 60 + periodSeconds: 2 + httpGet: + path: /healthz + port: task-api + readinessProbe: + periodSeconds: 2 + httpGet: + path: /readyz + port: task-api + livenessProbe: + periodSeconds: 2 + httpGet: + path: /healthz + port: task-api + resources: + requests: + cpu: 7000m + memory: 2Gi + - name: workflow-manager-helper + imagePullPolicy: Always + image: ghcr.io/nagare-media/engine/workflow-manager-helper:dev + args: [ "--config", "/config/workflow-manager-helper.yaml", "/run/secrets/engine.nagare.media/task/data.yaml" ] + ports: + - name: event-api + containerPort: 8181 + protocol: TCP + startupProbe: + failureThreshold: 60 + periodSeconds: 2 + httpGet: + path: /healthz + port: event-api + readinessProbe: + periodSeconds: 2 + httpGet: + path: /readyz + port: event-api + livenessProbe: + periodSeconds: 2 + httpGet: + path: /healthz + port: event-api + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: task-data + mountPath: /run/secrets/engine.nagare.media/task + readOnly: true + volumes: + - name: task-data + secret: + secretName: workflow-manager-helper-data-mmsys-test-scene-detection-test-recovery diff --git a/config/samples/k8s/mmsys-test-scene-detection/job-upload-media.yaml b/config/samples/k8s/mmsys-test-scene-detection/job-upload-media.yaml new file mode 100644 index 0000000..7975b5e --- /dev/null +++ b/config/samples/k8s/mmsys-test-scene-detection/job-upload-media.yaml @@ -0,0 +1,21 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: upload-media +spec: + template: + spec: + restartPolicy: Never # recreate pod by job controller + containers: + - name: upload + image: minio/mc:RELEASE.2023-02-28T00-12-59Z + command: ["/bin/sh", "-c"] + args: + - | + mc config host add s3 \ + http://minio.s3.svc.cluster.local:9000 \ + MLYEAE6AES7IUSL4APYL \ + snTrPIV2NDBoQOjpL10Nk6KY6EyWlePkF1NLMX3H \ + --api S3v4 + curl -sSL -o /tmp/caminandes-1-llama-drama.mp4 https://shio-eu-c-public.s3.eu-central-003.backblazeb2.com/media/testing/full-caminandes-1-llama-drama.mp4 + mc cp /tmp/caminandes-1-llama-drama.mp4 s3/nagare-media-engine-tests/inputs/caminandes-1-llama-drama.mp4 diff --git a/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-baseline.yaml b/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-baseline.yaml new file mode 100644 index 0000000..5193cfb --- /dev/null +++ b/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-baseline.yaml @@ -0,0 +1,40 @@ +apiVersion: engine.nagare.media/v1alpha1 +kind: WorkflowManagerHelperData + +workflow: + id: 38280525-1831-4f4d-b38b-de7e5e58e35b + humanReadable: + name: baseline + config: ~ + +task: + id: c774949a-6785-4d60-9add-98af8b161cd8 + humanReadable: + name: baseline + inputs: + - id: 0848ab75-2cee-4cfb-b94f-584cb4bd8313 + type: media + direction: pull + portBindings: + - id: input + url: http://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/inputs/caminandes-1-llama-drama.mp4 + metadata: + streams: + - duration: 1m30s + video: + frameRate: + average: 24 + outputs: + - id: e482adcf-ca7a-46ff-a945-ccb7aae039f4 + type: media + direction: push + portBindings: + - id: output + url: s3://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt?INSECURE=true&AWS_ACCESS_KEY_ID=MLYEAE6AES7IUSL4APYL&AWS_SECRET_ACCESS_KEY=snTrPIV2NDBoQOjpL10Nk6KY6EyWlePkF1NLMX3H + # -> http://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt + config: + mmsys-test-scene-detection.engine.nagare.media/test: baseline + +system: + nats: + url: nats://nats.nats.svc.cluster.local:4222 diff --git a/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-no-recovery.yaml b/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-no-recovery.yaml new file mode 100644 index 0000000..08ddeb8 --- /dev/null +++ b/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-no-recovery.yaml @@ -0,0 +1,50 @@ +apiVersion: engine.nagare.media/v1alpha1 +kind: WorkflowManagerHelperData + +workflow: + id: 6d0b3b44-af0f-47f3-a1fe-548794c1cee4 + humanReadable: + name: test-no-recovery + config: ~ + +task: + id: bdbfbb12-d14b-444e-8801-076b3ae162c6 + humanReadable: + name: test-no-recovery + inputs: + - id: c206ab16-028f-410b-af09-cd12657f00e1 + type: media + direction: pull + portBindings: + - id: input + url: http://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/inputs/caminandes-1-llama-drama.mp4 + metadata: + streams: + - duration: 1m30s + video: + frameRate: + average: 24 + - id: 4c23a4c2-2554-42f7-b2a2-e4cdadcc68eb + type: metadata + direction: push + portBindings: + - id: report + url: http://localhost:8080/events + metadata: + mimeType: application/cloudevents+json + outputs: + - id: 46ba0184-339c-4761-ab32-9219dfe27aab + type: media + direction: push + portBindings: + - id: output + url: s3://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt?INSECURE=true&AWS_ACCESS_KEY_ID=MLYEAE6AES7IUSL4APYL&AWS_SECRET_ACCESS_KEY=snTrPIV2NDBoQOjpL10Nk6KY6EyWlePkF1NLMX3H + # -> http://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt + config: + mmsys-test-scene-detection.engine.nagare.media/test: test-no-recovery + mmsys-test-scene-detection.engine.nagare.media/max-number-of-simulated-crashes: "1" + mmsys-test-scene-detection.engine.nagare.media/simulated-crash-wait-duration: 20s + +system: + nats: + url: nats://nats.nats.svc.cluster.local:4222 diff --git a/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-recovery.yaml b/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-recovery.yaml new file mode 100644 index 0000000..3087c7c --- /dev/null +++ b/config/samples/nagare-media/workflow-manager-helper-data_mmsys-test-scene-detection-test-recovery.yaml @@ -0,0 +1,50 @@ +apiVersion: engine.nagare.media/v1alpha1 +kind: WorkflowManagerHelperData + +workflow: + id: b2e6a8be-7512-414c-afef-239c5b009a98 + humanReadable: + name: test-recovery + config: ~ + +task: + id: a356af70-baed-488d-9fbd-77843221f5ef + humanReadable: + name: test-recovery + inputs: + - id: e4b47e85-f80b-426e-8d8d-32b5f834a2fc + type: media + direction: pull + portBindings: + - id: input + url: http://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/inputs/caminandes-1-llama-drama.mp4 + metadata: + streams: + - duration: 1m30s + video: + frameRate: + average: 24 + - id: f080d031-99ea-4dbc-a297-4015ce13f26d + type: metadata + direction: push + portBindings: + - id: report + url: http://localhost:8080/events + metadata: + mimeType: application/cloudevents+json + outputs: + - id: f6a65c06-716a-4d95-b777-58fac934015d + type: media + direction: push + portBindings: + - id: output + url: s3://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt?INSECURE=true&AWS_ACCESS_KEY_ID=MLYEAE6AES7IUSL4APYL&AWS_SECRET_ACCESS_KEY=snTrPIV2NDBoQOjpL10Nk6KY6EyWlePkF1NLMX3H + # -> http://minio.s3.svc.cluster.local:9000/nagare-media-engine-tests/outputs/caminandes-1-llama-drama.txt + config: + mmsys-test-scene-detection.engine.nagare.media/test: test-recovery + mmsys-test-scene-detection.engine.nagare.media/max-number-of-simulated-crashes: "1" + mmsys-test-scene-detection.engine.nagare.media/simulated-crash-wait-duration: 20s + +system: + nats: + url: nats://nats.nats.svc.cluster.local:4222 diff --git a/internal/functions/mmsys-test-scene-detection/func.go b/internal/functions/mmsys-test-scene-detection/func.go new file mode 100644 index 0000000..203b9a2 --- /dev/null +++ b/internal/functions/mmsys-test-scene-detection/func.go @@ -0,0 +1,618 @@ +/* +Copyright 2022-2024 The nagare media authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sleep + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/log" + + backoff "github.com/cenkalti/backoff/v4" + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + enginev1 "github.com/nagare-media/engine/api/v1alpha1" + "github.com/nagare-media/engine/internal/functions" + enginehttp "github.com/nagare-media/engine/internal/pkg/http" + "github.com/nagare-media/engine/internal/pkg/mime" + "github.com/nagare-media/engine/internal/pkg/uuid" + "github.com/nagare-media/engine/pkg/events" + "github.com/nagare-media/engine/pkg/nbmp" + nbmputils "github.com/nagare-media/engine/pkg/nbmp/utils" + nbmpv2 "github.com/nagare-media/models.go/iso/nbmp/v2" +) + +// Function description +const ( + Name = "mmsys-test-scene-detection" + + TestParameterKey = "mmsys-test-scene-detection.engine.nagare.media/test" + MaxNumberOfSimulatedCrashesParameterKey = "mmsys-test-scene-detection.engine.nagare.media/max-number-of-simulated-crashes" + SimulatedCrashWaitDurationParameterKey = "mmsys-test-scene-detection.engine.nagare.media/simulated-crash-wait-duration" + + DefaultMaxNumberOfSimulatedCrashes = 1 + DefaultSimulatedCrashWaitDuration = 20 * time.Second +) + +const ( + // * baseline: scene detection + // * test-no-recovery: scene detection with n disruptions + // * test-recovery: scene detection with n disruptions and recovery + + Baseline = "baseline" + TestNoRecovery = "test-no-recovery" + TestRecovery = "test-recovery" +) + +var ( + ffmpegScdetRegex = regexp.MustCompile(`.*scdet.*lavfi\.scd\.score:\D+(?P\d+\.\d+),.*lavfi\.scd\.time:\D+(?P