Merge branch 'main' into integrate-3.0.0rc20241115

build_tools/python_deploy/README.md

@@ -0,0 +1,48 @@
+# Python Deployment
+
+These scripts assist with building Python packages and pushing them to
+[PyPI (the Python Package Index)](https://pypi.org/). See also
+
+* The Python Packaging User Guide: <https://packaging.python.org/en/latest/>
+
+## Overview
+
+See comments in scripts for canonical usage. This page includes additional
+notes.
+
+### Package building
+
+These scripts build packages:
+
+* [`/shark-ai/build_tools/build_linux_package.sh`](/shark-ai/build_tools/build_linux_package.sh)
+* [`/sharktank/build_tools/build_linux_package.sh`](/sharktank/build_tools/build_linux_package.sh)
+* [`/shortfin/build_tools/build_linux_package.sh`](/shortfin/build_tools/build_linux_package.sh)
+
+### Version management
+
+These scripts handle versioning across packages, including considerations like
+major, minor, and patch levels (`X.Y.Z`), as well as suffixes like
+`rc20241107`:
+
+* [`compute_common_version.py`](./compute_common_version.py)
+* [`compute_local_version.py`](./compute_local_version.py)
+* [`promote_whl_from_rc_to_final.py`](./promote_whl_from_rc_to_final.py)
+* [`write_requirements.py`](./write_requirements.py)
+
+### PyPI deployment
+
+These scripts handle promoting nightly releases packages to stable and pushing
+to PyPI:
+
+* [`promote_whl_from_rc_to_final.py`](./promote_whl_from_rc_to_final.py)
+* [`pypi_deploy.sh`](./pypi_deploy.sh)
+
+Both of these scripts expect to have the dependencies from
+[`requirements-pypi-deploy.txt`](./requirements-pypi-deploy.txt) installed.
+This can be easily managed by using a Python virtual environment:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install -r ./requirements-pypi-deploy.txt
+```

build_tools/python_deploy/compute_common_version.py

@@ -6,8 +6,12 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 # This scripts grabs the `X.Y.Z[.dev]` version identifier from the
-# sharktank and shortfin version files and computes the version
-# for the meta package.
+# 'sharktank' and 'shortfin' version files and computes the version
+# for the meta 'shark-ai' package.
+#
+# Usage:
+#   ./compute_common_version.py --stable-release --write-json
+#   cat ../../shark-ai/version_local.json
 
 import argparse
 from pathlib import Path

build_tools/python_deploy/pypi_deploy.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+# Copyright 2024 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# This script promotes Python packages from nightly releases to PyPI.
+#
+# Prerequisites:
+#   * You will need to have PyPI credentials set up. See
+#     https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives
+#   * Install requirements, e.g. in a Python virtual environment (venv):
+#     `pip install -r requirements-pypi-deploy.txt`
+#   * Install python3.13t and install pip. On Ubuntu:
+#     ```bash
+#     sudo add-apt-repository ppa:deadsnakes
+#     sudo apt-get update
+#     sudo apt-get install python3.13-nogil
+#     python3.13t -m ensurepip --upgrade
+#     ```
+#   * Choose a release candidate to promote from
+#     https://github.com/nod-ai/SHARK-Platform/releases/tag/dev-wheels
+#
+# Usage:
+#   ./pypi_deploy.sh 2.9.0rc20241108
+
+set -euo pipefail
+
+RELEASE="$1"
+
+SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")";
+REPO_ROOT="$(cd "$SCRIPT_DIR"/../../ && pwd)"
+TMPDIR="$(mktemp --directory --tmpdir shark_platform_pypi_wheels.XXXXX)"
+ASSETS_PAGE="https://github.com/nod-ai/SHARK-Platform/releases/expanded_assets/dev-wheels"
+
+# TODO: rewrite in Python?
+
+function download_wheels() {
+  echo ""
+  echo "Downloading wheels for '${RELEASE}'..."
+
+  # sharktank
+  python -m pip download sharktank==${RELEASE} \
+    --no-deps --python-version 3.11 -f ${ASSETS_PAGE}
+
+  # shortfin
+  python -m pip download shortfin==${RELEASE} \
+    --no-deps --python-version 3.11 -f ${ASSETS_PAGE}
+  python -m pip download shortfin==${RELEASE} \
+    --no-deps --python-version 3.12 -f ${ASSETS_PAGE}
+  python -m pip download shortfin==${RELEASE} \
+    --no-deps --python-version 3.13 -f ${ASSETS_PAGE}
+  python -m pip download shortfin==${RELEASE} \
+    --no-deps --python-version 3.13 -f ${ASSETS_PAGE}
+  # TODO: fetch 3.13t using the same `python` somehow
+  #   * https://pip.pypa.io/en/stable/cli/pip_download/
+  #   * https://py-free-threading.github.io/installing_cpython/
+  #   * https://pip.pypa.io/en/stable/installation/
+  python3.13t -m pip download shortfin==${RELEASE} --no-deps -f ${ASSETS_PAGE}
+
+  # TODO: shark-ai meta package when it is published to nightlies
+
+  echo ""
+  echo "Downloaded wheels:"
+  ls
+}
+
+function edit_release_versions() {
+  echo ""
+  echo "Editing release versions..."
+  for file in *
+  do
+    ${SCRIPT_DIR}/promote_whl_from_rc_to_final.py ${file} --delete-old-wheel
+  done
+
+  echo "Edited wheels:"
+  ls
+}
+
+function upload_wheels() {
+  # TODO: list packages that would be uploaded, pause, prompt to continue
+  echo ""
+  echo "Uploading wheels:"
+  ls
+  twine upload --verbose *
+}
+
+function build_shark_ai_meta_package() {
+  # TODO: download meta package from nightly releases instead of this
+  #   Be aware that nightly releases pin other dependencies via the
+  #   generated `requirements.txt` compared to stable releases.
+  echo ""
+
+  # TODO: rework `write_requirements.py` to use the versions from the downloaded whls?
+  echo "Computing local versions for sharktank and shortfin..."
+  ${SCRIPT_DIR}/compute_local_version.py ${REPO_ROOT}/sharktank
+  ${SCRIPT_DIR}/compute_local_version.py ${REPO_ROOT}/shortfin
+
+  echo "Computing common version for shark-ai meta package..."
+  ${SCRIPT_DIR}/compute_common_version.py --stable-release --write-json
+
+  echo "Writing requirements for shark-ai meta package..."
+  ${SCRIPT_DIR}/write_requirements.py
+
+  echo "Building shark-ai meta package..."
+  ${REPO_ROOT}/shark-ai/build_tools/build_linux_package.sh
+
+  # TODO: This is error-prone. We only want to publish the whl for this release.
+  #   Copy instead? Specify exact file name? Clear directory before building?
+  mv ${REPO_ROOT}/shark-ai/build_tools/wheelhouse/* .
+}
+
+function main() {
+  echo "Changing into ${TMPDIR}"
+  cd "${TMPDIR}"
+  # TODO: check_requirements (using pip)
+
+  download_wheels
+  edit_release_versions
+  build_shark_ai_meta_package
+  upload_wheels
+}
+
+main

shark-ai/README.md

@@ -1,3 +1,3 @@
 # SHARK AI meta package
 
-Meta package to install `sharktank` and `shortfin`.
+Meta package to install `shortfin` and compatible IREE packages.

shark-ai/build_tools/build_linux_package.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2024 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# build_linux_package.sh
+#
+# Builds shark-ai Python package for Linux.
+#
+# Usage:
+#   ./build_tools/build_linux_package.sh
+
+set -xeu -o errtrace
+
+THIS_DIR="$(cd $(dirname $0) && pwd)"
+REPO_ROOT="$(cd "$THIS_DIR"/../../ && pwd)"
+OUTPUT_DIR="${OUTPUT_DIR:-${THIS_DIR}/wheelhouse}"
+
+python -m pip wheel --disable-pip-version-check --no-deps -v -w "${OUTPUT_DIR}" "${REPO_ROOT}/shark-ai"
+
+wheel_output="$(echo "${OUTPUT_DIR}/shark_ai-"*".whl")"
+ls "${wheel_output}"