From 91335dfc5f3167e65c185137b715f61e21b4f4ce Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 28 Nov 2024 15:27:49 +0200 Subject: [PATCH 1/9] kfp-launcher rock and test --- launcher/rockcraft.yaml | 76 +++++++++++++++++++++++++++++++++++++ launcher/tests/test_rock.py | 36 ++++++++++++++++++ launcher/tox.ini | 53 ++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 launcher/rockcraft.yaml create mode 100644 launcher/tests/test_rock.py create mode 100644 launcher/tox.ini diff --git a/launcher/rockcraft.yaml b/launcher/rockcraft.yaml new file mode 100644 index 0000000..ef95ad1 --- /dev/null +++ b/launcher/rockcraft.yaml @@ -0,0 +1,76 @@ +# Based on: https://github.com/kubeflow/pipelines/blob/2.2.0/backend/Dockerfile.launcher +name: kfp-launcher +summary: Kubeflow Pipelines Launcher +description: | + Kubeflow Pipelines Launcher is a streamlined interface designed to simplify the creation, management, and deployment of machine learning workflows on Kubernetes. +version: v2.2.0 +license: Apache-2.0 +base: ubuntu@22.04 +run-user: _daemon_ +platforms: + amd64: + +services: + launcher: + override: merge + summary: "kfp launcher service" + startup: enabled + user: appuser + command: "/bin/launcher-v2" + +parts: + security-team-requirement: + plugin: nil + override-build: | + mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks + (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \ + dpkg-query -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) > \ + ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query + + launcher: + plugin: go + source-type: git + source: https://github.com/kubeflow/pipelines.git + source-depth: 1 + source-tag: 2.2.0 + build-snaps: + - go/1.21/stable + build-packages: + - apt + - bash + build-environment: + - CGO_ENABLED: 0 + - GOOS: linux + - GOARCH: amd64 + stage-packages: + - bash + override-build: | + set -xe + + go build -tags netgo -ldflags '-extldflags "-static"' -o launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go + + ./hack/install-go-licenses.sh + + $GOBIN/go-licenses check $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 + $GOBIN/go-licenses csv $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 > licenses.csv && \ + diff licenses.csv $CRAFT_PART_BUILD/backend/third_party_licenses/launcher.csv && \ + $GOBIN/go-licenses save $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 --save_path NOTICES + + override-prime: | + mkdir -p $CRAFT_PRIME/bin/third_party + cp -r $CRAFT_PART_BUILD/launcher-v2 $CRAFT_PRIME/bin + cp -r $CRAFT_PART_BUILD/licenses.csv $CRAFT_PRIME/bin/third_party + cp -r $CRAFT_PART_BUILD/NOTICES $CRAFT_PRIME/bin/third_party + + # not-root user for this rock should be 'appuser' + non-root-user: + plugin: nil + after: [ launcher ] + overlay-script: | + # Create a user in the $CRAFT_OVERLAY chroot + groupadd -R $CRAFT_OVERLAY -g 1001 appuser + useradd -R $CRAFT_OVERLAY -M -r -u 1001 -g appuser appuser + override-prime: | + craftctl default + chown -R 584792:users bin + \ No newline at end of file diff --git a/launcher/tests/test_rock.py b/launcher/tests/test_rock.py new file mode 100644 index 0000000..70cf3f4 --- /dev/null +++ b/launcher/tests/test_rock.py @@ -0,0 +1,36 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import pytest +import subprocess + +from charmed_kubeflow_chisme.rock import CheckRock + + +@pytest.mark.abort_on_fail +def test_rock(): + """Test rock.""" + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() + LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" + + # assert the rock contains the expected files + subprocess.run( + [ + "docker", + "run", + "--rm", + LOCAL_ROCK_IMAGE, + "exec", + "ls", + "-la", + "/bin/third_party", + ], + check=True, + ) + + subprocess.run( + ["docker", "run", "--rm", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/bin/launcher-v2"], + check=True, + ) diff --git a/launcher/tox.ini b/launcher/tox.ini new file mode 100644 index 0000000..eab50eb --- /dev/null +++ b/launcher/tox.ini @@ -0,0 +1,53 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +[tox] +skipsdist = True +skip_missing_interpreters = True +envlist = unit, sanity, integration + +[testenv] +setenv = + PYTHONPATH={toxinidir} + PYTHONBREAKPOINT=ipdb.set_trace + CHARM_REPO=https://github.com/canonical/kfp-operators.git + CHARM_BRANCH=main + LOCAL_CHARM_DIR=charm_repo + +[testenv:pack] +passenv = * +allowlist_externals = + rockcraft +commands = + rockcraft pack + +[testenv:export-to-docker] +passenv = * +allowlist_externals = + bash + skopeo + yq +commands = + # export to docker + bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ + VERSION=$(yq eval .version rockcraft.yaml) && \ + ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \ + DOCKER_IMAGE=$NAME:$VERSION && \ + echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \ + skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE' + +[testenv:sanity] +passenv = * +deps = + pytest + charmed-kubeflow-chisme +commands = + pytest -s -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests + +[testenv:integration] +passenv = * +allowlist_externals = + echo +commands = + echo "WARNING: This is a placeholder test - no test is implemented here." + From fcd0dd90bb68db266c5e7cfda68f5ea2752dcf39 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 28 Nov 2024 15:36:05 +0200 Subject: [PATCH 2/9] kfp-launcher rock and test --- launcher/tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launcher/tox.ini b/launcher/tox.ini index eab50eb..f802cb3 100644 --- a/launcher/tox.ini +++ b/launcher/tox.ini @@ -3,7 +3,7 @@ [tox] skipsdist = True skip_missing_interpreters = True -envlist = unit, sanity, integration +envlist = pack, export-to-docker, unit, sanity, integration [testenv] setenv = @@ -51,3 +51,4 @@ allowlist_externals = commands = echo "WARNING: This is a placeholder test - no test is implemented here." +> \ No newline at end of file From f3f41aea7436a369c40b02c383b01ee301eaeb30 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 28 Nov 2024 15:40:37 +0200 Subject: [PATCH 3/9] kfp-launcher rock and test --- launcher/tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/launcher/tox.ini b/launcher/tox.ini index f802cb3..da8f12b 100644 --- a/launcher/tox.ini +++ b/launcher/tox.ini @@ -49,6 +49,5 @@ passenv = * allowlist_externals = echo commands = - echo "WARNING: This is a placeholder test - no test is implemented here." - -> \ No newline at end of file + # TODO: Implement integration tests here + echo "WARNING: This is a placeholder test - no test is implemented here." \ No newline at end of file From e46391abda4f33998e2bbe6b7f275f0077d362ab Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 29 Nov 2024 10:58:54 +0200 Subject: [PATCH 4/9] kfp-launcher rock and test --- launcher/rockcraft.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launcher/rockcraft.yaml b/launcher/rockcraft.yaml index ef95ad1..755804e 100644 --- a/launcher/rockcraft.yaml +++ b/launcher/rockcraft.yaml @@ -57,10 +57,10 @@ parts: $GOBIN/go-licenses save $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 --save_path NOTICES override-prime: | - mkdir -p $CRAFT_PRIME/bin/third_party + mkdir -p $CRAFT_PRIME/third_party cp -r $CRAFT_PART_BUILD/launcher-v2 $CRAFT_PRIME/bin - cp -r $CRAFT_PART_BUILD/licenses.csv $CRAFT_PRIME/bin/third_party - cp -r $CRAFT_PART_BUILD/NOTICES $CRAFT_PRIME/bin/third_party + cp -r $CRAFT_PART_BUILD/licenses.csv $CRAFT_PRIME/third_party + cp -r $CRAFT_PART_BUILD/NOTICES $CRAFT_PRIME/third_party # not-root user for this rock should be 'appuser' non-root-user: From c7283c357e7d1b0883a9bac2bf28be616346fc38 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 29 Nov 2024 12:47:04 +0200 Subject: [PATCH 5/9] kfp-launcher rock and test --- launcher/rockcraft.yaml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/launcher/rockcraft.yaml b/launcher/rockcraft.yaml index 755804e..0e9415e 100644 --- a/launcher/rockcraft.yaml +++ b/launcher/rockcraft.yaml @@ -46,21 +46,18 @@ parts: - bash override-build: | set -xe + + mkdir -p $CRAFT_PART_INSTALL/bin + mkdir -p $CRAFT_PART_INSTALL/third_party - go build -tags netgo -ldflags '-extldflags "-static"' -o launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go + go build -tags netgo -ldflags '-extldflags "-static"' -o $CRAFT_PART_INSTALL/bin/launcher-v2 $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2/*.go ./hack/install-go-licenses.sh $GOBIN/go-licenses check $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 - $GOBIN/go-licenses csv $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 > licenses.csv && \ - diff licenses.csv $CRAFT_PART_BUILD/backend/third_party_licenses/launcher.csv && \ - $GOBIN/go-licenses save $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 --save_path NOTICES - - override-prime: | - mkdir -p $CRAFT_PRIME/third_party - cp -r $CRAFT_PART_BUILD/launcher-v2 $CRAFT_PRIME/bin - cp -r $CRAFT_PART_BUILD/licenses.csv $CRAFT_PRIME/third_party - cp -r $CRAFT_PART_BUILD/NOTICES $CRAFT_PRIME/third_party + $GOBIN/go-licenses csv $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 > $CRAFT_PART_INSTALL/third_party/licenses.csv && \ + diff $CRAFT_PART_INSTALL/third_party/licenses.csv $CRAFT_PART_BUILD/backend/third_party_licenses/launcher.csv && \ + $GOBIN/go-licenses save $CRAFT_PART_BUILD/backend/src/v2/cmd/launcher-v2 --save_path $CRAFT_PART_INSTALL/third_party/NOTICES # not-root user for this rock should be 'appuser' non-root-user: From 6508de30f403a0a3fb7b84faaf9bf8bf926c6c50 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Sat, 30 Nov 2024 08:00:25 +0200 Subject: [PATCH 6/9] kfp-launcher rock and test --- launcher/tests/test_rock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/tests/test_rock.py b/launcher/tests/test_rock.py index 70cf3f4..8ae8afc 100644 --- a/launcher/tests/test_rock.py +++ b/launcher/tests/test_rock.py @@ -25,7 +25,7 @@ def test_rock(): "exec", "ls", "-la", - "/bin/third_party", + "/third_party", ], check=True, ) From ac8c942309a2d5e9e05ed594de7d2edb048cab61 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Sat, 30 Nov 2024 08:06:44 +0200 Subject: [PATCH 7/9] kfp-launcher rock and test --- launcher/tests/test_rock.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/launcher/tests/test_rock.py b/launcher/tests/test_rock.py index 8ae8afc..a888aa8 100644 --- a/launcher/tests/test_rock.py +++ b/launcher/tests/test_rock.py @@ -30,6 +30,21 @@ def test_rock(): check=True, ) + # Assert the rock contains the expected file in /third_party + subprocess.run( + [ + "docker", + "run", + "--rm", + LOCAL_ROCK_IMAGE, + "exec", + "ls", + "-la", + "/third_party/NOTICES", + ], + check=True, + ) + subprocess.run( ["docker", "run", "--rm", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/bin/launcher-v2"], check=True, From 338a4d50f4e7b2edcfcb10e912ab8e43f0fb49b2 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Sat, 30 Nov 2024 08:09:05 +0200 Subject: [PATCH 8/9] kfp-launcher rock and test --- launcher/tests/test_rock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launcher/tests/test_rock.py b/launcher/tests/test_rock.py index a888aa8..b746f02 100644 --- a/launcher/tests/test_rock.py +++ b/launcher/tests/test_rock.py @@ -25,7 +25,7 @@ def test_rock(): "exec", "ls", "-la", - "/third_party", + "/third_party/licenses.csv", ], check=True, ) @@ -40,7 +40,7 @@ def test_rock(): "exec", "ls", "-la", - "/third_party/NOTICES", + "/third_party/NOTICES/", ], check=True, ) From 8a29cf5abb4d7784f013cb251fe798d774f95466 Mon Sep 17 00:00:00 2001 From: BON4 Date: Tue, 3 Dec 2024 10:26:19 +0200 Subject: [PATCH 9/9] Update launcher/rockcraft.yaml Change version to correct one Co-authored-by: Daniela Plascencia --- launcher/rockcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/rockcraft.yaml b/launcher/rockcraft.yaml index 0e9415e..df0160f 100644 --- a/launcher/rockcraft.yaml +++ b/launcher/rockcraft.yaml @@ -3,7 +3,7 @@ name: kfp-launcher summary: Kubeflow Pipelines Launcher description: | Kubeflow Pipelines Launcher is a streamlined interface designed to simplify the creation, management, and deployment of machine learning workflows on Kubernetes. -version: v2.2.0 +version: 2.2.0 license: Apache-2.0 base: ubuntu@22.04 run-user: _daemon_