From 8af6d3c3b0d48023a1fa6948e210f7cebfee363e Mon Sep 17 00:00:00 2001 From: Michal Hucko Date: Wed, 29 May 2024 12:34:56 +0200 Subject: [PATCH] Update persistenceagent to 2.2.0 (#89) * Update persistenceagent to 2.2.0 --- persistenceagent/rockcraft.yaml | 32 +++++++------ persistenceagent/tests/test_rock.py | 73 +++++++++++++++++++++++++++++ persistenceagent/tox.ini | 14 +++--- 3 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 persistenceagent/tests/test_rock.py diff --git a/persistenceagent/rockcraft.yaml b/persistenceagent/rockcraft.yaml index 40c74db..e6e987c 100644 --- a/persistenceagent/rockcraft.yaml +++ b/persistenceagent/rockcraft.yaml @@ -1,11 +1,13 @@ -# Dockerfile: https://github.com/kubeflow/pipelines/blob/2.0.5/backend/Dockerfile.persistenceagent +# Based on: https://github.com/kubeflow/pipelines/blob/2.2.0/backend/Dockerfile.persistenceagent name: persistenceagent summary: Reusable end-to-end ML workflows built using the Kubeflow Pipelines SDK description: | This component serves as the backend persistence agent of Kubeflow pipelines. -version: 2.0.5 +version: "2.2.0" license: Apache-2.0 -base: ubuntu:22.04 +base: ubuntu@22.04 +platforms: + amd64: run-user: _daemon_ services: persistenceagent: @@ -16,18 +18,26 @@ services: NAMESPACE: "" TTL_SECONDS_AFTER_WORKFLOW_FINISH: 86400 NUM_WORKERS: 2 - command: bash -c 'persistence_agent --logtostderr=true --namespace=$NAMESPACE --ttlSecondsAfterWorkflowFinish=$TTL_SECONDS_AFTER_WORKFLOW_FINISH --numWorker $NUM_WORKERS' -platforms: - amd64: + LOG_LEVEL: "info" + EXECUTIONTYPE: "Workflow" + command: bash -c 'persistence_agent --logtostderr=true --namespace=$NAMESPACE --ttlSecondsAfterWorkflowFinish=$TTL_SECONDS_AFTER_WORKFLOW_FINISH --numWorker $NUM_WORKERS --executionType $EXECUTIONTYPE --logLevel=$LOG_LEVEL' parts: + security-team-requirement: + plugin: nil + override-build: | + mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks + (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \ + dpkg-query --root=${CRAFT_PROJECT_DIR}/../bundles/ubuntu-22.04/rootfs/ -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) \ + > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query + persistenceagent: plugin: go source: https://github.com/kubeflow/pipelines source-type: git - source-tag: 2.0.5 + source-tag: 2.2.0 build-snaps: - - go/1.20/stable + - go/1.21/stable build-packages: - git - openssl @@ -43,9 +53,3 @@ parts: $GOBIN/go-licenses csv ./backend/src/agent/persistence > $CRAFT_PART_INSTALL/third_party/licenses.csv && \ diff $CRAFT_PART_INSTALL/third_party/licenses.csv backend/third_party_licenses/persistence_agent.csv && \ $GOBIN/go-licenses save ./backend/src/agent/persistence --save_path $CRAFT_PART_INSTALL/third_party/NOTICES - - # security requirement - mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks - (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \ - dpkg-query --root=${CRAFT_PROJECT_DIR}/../bundles/ubuntu-22.04/rootfs/ -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) \ - > ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query diff --git a/persistenceagent/tests/test_rock.py b/persistenceagent/tests/test_rock.py new file mode 100644 index 0000000..187747f --- /dev/null +++ b/persistenceagent/tests/test_rock.py @@ -0,0 +1,73 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +import random +import pytest +import string +import subprocess +import yaml + +from charmed_kubeflow_chisme.rock import CheckRock + + +@pytest.fixture() +def rock_test_env(tmpdir): + """Yields a temporary directory and random docker container name, then cleans them up after.""" + container_name = "".join( + [str(i) for i in random.choices(string.ascii_lowercase, k=8)] + ) + yield tmpdir, container_name + + try: + subprocess.run(["docker", "rm", container_name]) + except Exception: + pass + # tmpdir fixture we use here should clean up the other files for us + + +@pytest.mark.abort_on_fail +def test_rock(rock_test_env): + """Test rock.""" + temp_dir, container_name = rock_test_env + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() + LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" + + # assert we have the expected files + subprocess.run( + [ + "docker", + "run", + "--entrypoint", + "/bin/bash", + LOCAL_ROCK_IMAGE, + "-c", + "which persistence_agent", + ], + check=True, + ) + subprocess.run( + [ + "docker", + "run", + "--entrypoint", + "/bin/bash", + LOCAL_ROCK_IMAGE, + "-c", + "ls -la /usr/bin/persistence_agent", + ], + check=True, + ) + subprocess.run( + [ + "docker", + "run", + "--entrypoint", + "/bin/bash", + LOCAL_ROCK_IMAGE, + "-c", + "ls -la /third_party", + ], + check=True, + ) + diff --git a/persistenceagent/tox.ini b/persistenceagent/tox.ini index 9b969b6..329f84b 100644 --- a/persistenceagent/tox.ini +++ b/persistenceagent/tox.ini @@ -1,4 +1,4 @@ -# Copyright 2022 Canonical Ltd. +# Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. [tox] skipsdist = True @@ -38,11 +38,12 @@ commands = [testenv:sanity] passenv = * -allowlist_externals = - echo +deps = + pytest + charmed-kubeflow-chisme commands = - # TODO: Implement sanity tests - echo "WARNING: This is a placeholder test - no test is implemented here." + # run rock tests + pytest -s -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests [testenv:integration] passenv = * @@ -59,8 +60,7 @@ deps = ops commands = echo "WARNING: This is a placeholder test - no test is implemented here." - ; Remove warning and uncomment below once https://github.com/canonical/kfp-operators/issues/377 - ; is resolved + # Tests bellow are failing (charms are unable to build) ; # clone related charm ; rm -rf {env:LOCAL_CHARM_DIR} ; git clone --branch {env:CHARM_BRANCH} {env:CHARM_REPO} {env:LOCAL_CHARM_DIR}