From 2a80c7b5837b2918619c8c1761669e3635754f50 Mon Sep 17 00:00:00 2001 From: BON4 Date: Mon, 13 Jan 2025 14:08:19 +0200 Subject: [PATCH] Added Ml metadata server rock image (#160) * ml-metadata-store-server rock * ml-metadata-store-server rock and test * ml-metadata-store-server rock and test * ml-metadata-store-server rock and test * ml-metadata-store-server rock and test * ml-metadata-store-server rock and test * ml-metadata-store-server rock and test * Update ml-metadata/rockcraft.yaml Co-authored-by: Noha Ihab <49988746+NohaIhab@users.noreply.github.com> --------- Co-authored-by: Noha Ihab <49988746+NohaIhab@users.noreply.github.com> --- ml-metadata/rockcraft.yaml | 79 ++++++++++++++++++++++++++++++++++ ml-metadata/tests/test_rock.py | 36 ++++++++++++++++ ml-metadata/tox.ini | 53 +++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 ml-metadata/rockcraft.yaml create mode 100644 ml-metadata/tests/test_rock.py create mode 100644 ml-metadata/tox.ini diff --git a/ml-metadata/rockcraft.yaml b/ml-metadata/rockcraft.yaml new file mode 100644 index 0000000..80d9877 --- /dev/null +++ b/ml-metadata/rockcraft.yaml @@ -0,0 +1,79 @@ +# Based on: https://github.com/google/ml-metadata/blob/v1.14.0/ml_metadata/tools/docker_server/Dockerfile +name: ml-metadata +summary: Part of the ML Metadata (MLMD) framework. +description: | + ML Metadata is a framework for managing and tracking metadata associated with ML workflows. + It is part of the TensorFlow Extended (TFX) ecosystem, but it can be used independently as well. +version: "1.14.0" +license: Apache-2.0 +# Build does not work with ubuntu@22.04. Issue: https://github.com/google/ml-metadata/issues/210 +base: ubuntu@20.04 +run-user: _daemon_ +platforms: + amd64: + +services: + mlmd: + override: replace + summary: "ml metadata store service" + startup: enabled + environment: + GRPC_PORT: "8080" + METADATA_STORE_SERVER_CONFIG_FILE: "" + command: "/bin/metadata_store_server --grpc_port=${GRPC_PORT} --metadata_store_server_config_file=${METADATA_STORE_SERVER_CONFIG_FILE}" + +parts: + security-team-requirement: + plugin: nil + override-build: | + mkdir -p ${CRAFT_PART_INSTALL}/usr/share/rocks + (echo "# os-release" && cat /etc/os-release && echo "# dpkg-query" && \ + dpkg-query -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W) > \ + ${CRAFT_PART_INSTALL}/usr/share/rocks/dpkg.query + + mlmd: + plugin: nil + source: https://github.com/google/ml-metadata.git + source-tag: v1.14.0 + build-packages: + - build-essential + - clang + - cmake + - make + - musl-dev + - ca-certificates + - openssl + - curl + - unzip + - software-properties-common + - git + - python3-distutils + - python-is-python3 + - python3-dev + stage-packages: + - tzdata + build-environment: + - BAZEL_VERSION: 5.3.0 + override-build: | + set -xe + + curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh + + curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE + + chmod +x bazel-*.sh + + ./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh + + cd $CRAFT_PART_SRC + + bazel clean --expunge + + bazel build -c opt --action_env=PATH --define=grpc_no_ares=true //ml_metadata/metadata_store:metadata_store_server --cxxopt="-std=c++17" + + mkdir -p $CRAFT_PART_INSTALL/bin + mkdir -p $CRAFT_PART_INSTALL/third_party + + cp -RL $CRAFT_PART_SRC/bazel-src/external/libmysqlclient $CRAFT_PART_INSTALL/third_party/mariadb-connector-c + + cp $CRAFT_PART_SRC/bazel-bin/ml_metadata/metadata_store/metadata_store_server $CRAFT_PART_INSTALL/bin/metadata_store_server diff --git a/ml-metadata/tests/test_rock.py b/ml-metadata/tests/test_rock.py new file mode 100644 index 0000000..8d3f6c9 --- /dev/null +++ b/ml-metadata/tests/test_rock.py @@ -0,0 +1,36 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import pytest +import subprocess + +from charmed_kubeflow_chisme.rock import CheckRock + + +@pytest.mark.abort_on_fail +def test_rock(): + """Test rock.""" + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() + LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" + + # assert the rock contains the expected files + subprocess.run( + [ + "docker", + "run", + "--rm", + LOCAL_ROCK_IMAGE, + "exec", + "ls", + "-la", + "/third_party/mariadb-connector-c", + ], + check=True, + ) + + subprocess.run( + ["docker", "run", "--rm", LOCAL_ROCK_IMAGE, "exec", "ls", "-la", "/bin/metadata_store_server"], + check=True, + ) diff --git a/ml-metadata/tox.ini b/ml-metadata/tox.ini new file mode 100644 index 0000000..a1b45b9 --- /dev/null +++ b/ml-metadata/tox.ini @@ -0,0 +1,53 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +[tox] +skipsdist = True +skip_missing_interpreters = True +envlist = pack, export-to-docker, unit, sanity, integration + +[testenv] +setenv = + PYTHONPATH={toxinidir} + PYTHONBREAKPOINT=ipdb.set_trace + CHARM_REPO=https://github.com/canonical/mlmd-operator.git + CHARM_BRANCH=main + LOCAL_CHARM_DIR=charm_repo + +[testenv:pack] +passenv = * +allowlist_externals = + rockcraft +commands = + rockcraft pack + +[testenv:export-to-docker] +passenv = * +allowlist_externals = + bash + skopeo + yq +commands = + # export to docker + bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ + VERSION=$(yq eval .version rockcraft.yaml) && \ + ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \ + DOCKER_IMAGE=$NAME:$VERSION && \ + echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \ + skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE' + +[testenv:sanity] +passenv = * +deps = + pytest + charmed-kubeflow-chisme +commands = + pytest -s -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests + +[testenv:integration] +passenv = * +allowlist_externals = + echo +commands = + # TODO: Implement integration tests here + echo "WARNING: This is a placeholder test - no test is implemented here."