Add aarch64 Linux wheels to build / release workflow (#109)

.dockerignore

@@ -0,0 +1,3 @@
+.idea
+.git
+__pycache__

.github/docker/Dockerfile

@@ -0,0 +1,36 @@
+# syntax=docker/dockerfile:1.4
+
+ARG PYTHON_TAG
+FROM python:${PYTHON_TAG} as build
+
+ARG DEBIAN_FRONTEND="noninteractive"
+ARG TARGETARCH
+ARG PYTHON_TAG
+RUN \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-${TARGETARCH}-${PYTHON_TAG} \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked,id=apt-${TARGETARCH}-${PYTHON_TAG} \
+    set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache; \
+    wget --no-hsts -O /usr/share/keyrings/adoptium.asc https://packages.adoptium.net/artifactory/api/gpg/key/public; \
+    echo "deb [signed-by=/usr/share/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list; \
+    apt-get -qq update; \
+    apt-get -qq -y --no-install-recommends install temurin-8-jdk maven
+RUN \
+    --mount=type=cache,target=/root/.cache/pip,sharing=locked \
+    set -eux; \
+    python -m venv /jpy-build-venv; \
+    /jpy-build-venv/bin/pip install --upgrade pip setuptools; \
+    /jpy-build-venv/bin/pip install --only-binary=:all: wheel patchelf
+ENV JAVA_HOME=/usr/lib/jvm/temurin-8-jdk-${TARGETARCH} \
+    VIRTUAL_ENV='/jpy-build-venv' \
+    PATH="/jpy-build-venv/bin:${PATH}" \
+    CI=true
+COPY --link . /jpy
+RUN \
+  --mount=type=cache,target=/root/.m2,sharing=locked \
+  cd /jpy; \
+  .github/env/Linux/bdist-wheel.sh
+
+FROM scratch
+COPY --link --from=build /jpy/dist/* .

.github/docker/README.md

@@ -0,0 +1,19 @@
+# docker buildx bake
+
+This is an alternative way to build the jpy Linux wheels.
+It is mainly used as a means to produce arm64 wheels via QEMU cross-compiling since GHA does not have a native Linux arm64 runner.
+
+That said, it can be useful locally too; either to produce native or arm64 Linux wheels.
+
+```bash
+docker buildx bake \
+  --file .github/docker/docker-bake.hcl \
+  --set "*.output=type=local,dest=/tmp/dist"
+```
+
+```bash
+docker buildx bake \
+  --file .github/docker/docker-bake.hcl \
+  --set "*.output=type=local,dest=/tmp/dist" \
+  --set "*.platform=linux/arm64/v8" 
+```

.github/docker/docker-bake.hcl

@@ -0,0 +1,70 @@
+group "default" {
+    targets = [
+        "python-36-linux",
+        "python-37-linux",
+        "python-38-linux",
+        "python-39-linux",
+        "python-310-linux",
+        "python-311-linux"
+    ]
+}
+
+variable "DEBIAN_BASE" {
+    default = "bullseye"
+}
+
+variable "GITHUB_ACTIONS" {
+    default = false
+}
+
+target "shared" {
+    dockerfile = ".github/docker/Dockerfile"
+    cache-from = [
+        GITHUB_ACTIONS ? "type=gha,scope=jpy-build" : ""
+    ]
+    cache-to = [
+        GITHUB_ACTIONS ? "type=gha,mode=max,scope=jpy-build" : ""
+    ]
+}
+
+target "python-36-linux" {
+    inherits = [ "shared" ]
+    args = {
+        PYTHON_TAG = "3.6-${DEBIAN_BASE}"
+    }
+}
+
+target "python-37-linux" {
+    inherits = [ "shared" ]
+    args = {
+        PYTHON_TAG = "3.7-${DEBIAN_BASE}"
+    }
+}
+
+target "python-38-linux" {
+    inherits = [ "shared" ]
+    args = {
+        PYTHON_TAG = "3.8-${DEBIAN_BASE}"
+    }
+}
+
+target "python-39-linux" {
+    inherits = [ "shared" ]
+    args = {
+        PYTHON_TAG = "3.9-${DEBIAN_BASE}"
+    }
+}
+
+target "python-310-linux" {
+    inherits = [ "shared" ]
+    args = {
+        PYTHON_TAG = "3.10-${DEBIAN_BASE}"
+    }
+}
+
+target "python-311-linux" {
+    inherits = [ "shared" ]
+    args = {
+        PYTHON_TAG = "3.11-${DEBIAN_BASE}"
+    }
+}

.github/env/Linux/bdist-wheel.sh

@@ -4,12 +4,17 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-python -m pip install -r .github/env/Linux/requirements.txt
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+python -m pip install -r "${__dir}/requirements.txt"
+
+# Can add --build-number <x> if necessary
 python setup.py bdist_wheel --dist-dir dist.linux
+
 # Note: auditwheel only works with a single file argument - we are relying on finding exactly one wheel
-python .github/env/Linux/auditwheel-keep-libjvm.py \
+python "${__dir}/auditwheel-keep-libjvm.py" \
   repair \
-  --plat manylinux_2_17_x86_64 \
+  --plat "manylinux_2_17_$(arch)" \
   --only-plat \
   --wheel-dir dist/ \
   dist.linux/*

.github/workflows/build.yml

@@ -62,6 +62,8 @@ jobs:
           - { machine: 'macos-11', python: '3.9', pythonArchitecture: 'x64', arch: 'amd64', cmd: '.github/env/macOS/bdist-wheel.sh' }
           - { machine: 'macos-11', python: '3.10', pythonArchitecture: 'x64', arch: 'amd64', cmd: '.github/env/macOS/bdist-wheel.sh' }
           - { machine: 'macos-11', python: '3.11', pythonArchitecture: 'x64', arch: 'amd64', cmd: '.github/env/macOS/bdist-wheel.sh' }
+          # Add M1 powered runners when available
+          # https://github.com/jpy-consortium/jpy/issues/110
 
     steps:
       - uses: actions/checkout@v4
@@ -85,17 +87,43 @@ jobs:
           path: dist/*.whl
           retention-days: 1
 
+  bdist-wheels-linux-arm64:
+    runs-on: 'ubuntu-22.04'
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Bake
+        uses: docker/bake-action@v3.1.0
+        with:
+          files: .github/docker/docker-bake.hcl
+          pull: true
+          set: |
+            *.platform=linux/arm64/v8
+            *.output=type=local,dest=/tmp/dist
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: bdist-wheels-linux-arm64
+          path: /tmp/dist/*.whl
+          retention-days: 1
+
   collect-artifacts:
     runs-on: ubuntu-22.04
-    needs: ['sdist', 'bdist-wheel']
+    needs: ['sdist', 'bdist-wheel', 'bdist-wheels-linux-arm64']
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
         with:
           path: download-artifacts
       - name: collect-artifacts.sh
         run: .github/scripts/collect-artifacts.sh
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: jpy
           path: collect-artifacts

RELEASE.md

@@ -0,0 +1,45 @@
+# Release
+
+Jpy is built and released across a matrix of operating systems, architectures, and Python versions.
+
+The jar artifacts are Java 8+ compatible and released to [org.jpyconsortium:jpy](https://repo1.maven.org/maven2/org/jpyconsortium/jpy) on Maven Central.
+
+The wheel artifacts are compatible with the table below, and released to the PyPi package [jpy](https://pypi.org/project/jpy/).
+
+| Version | OS      | Arch   |
+|---------|---------|--------|
+| 3.6     | Linux   | x86_64 |
+| 3.7     | Linux   | x86_64 |
+| 3.8     | Linux   | x86_64 |
+| 3.9     | Linux   | x86_64 |
+| 3.10    | Linux   | x86_64 |
+| 3.11    | Linux   | x86_64 |
+| 3.6     | Linux   | arm64  |
+| 3.7     | Linux   | arm64  |
+| 3.8     | Linux   | arm64  |
+| 3.9     | Linux   | arm64  |
+| 3.10    | Linux   | arm64  |
+| 3.11    | Linux   | arm64  |
+| 3.6     | MacOS   | x86_64 |
+| 3.7     | MacOS   | x86_64 |
+| 3.8     | MacOS   | x86_64 |
+| 3.9     | MacOS   | x86_64 |
+| 3.10    | MacOS   | x86_64 |
+| 3.11    | MacOS   | x86_64 |
+| 3.6     | MacOS   | arm64  |
+| 3.7     | MacOS   | arm64  |
+| 3.8     | MacOS   | arm64  |
+| 3.9     | MacOS   | arm64  |
+| 3.10    | MacOS   | arm64  |
+| 3.11    | MacOS   | arm64  |
+| 3.6     | Windows | x86_64 |
+| 3.7     | Windows | x86_64 |
+| 3.8     | Windows | x86_64 |
+| 3.9     | Windows | x86_64 |
+| 3.10    | Windows | x86_64 |
+| 3.11    | Windows | x86_64 |
+
+## Process
+
+The [build.yml](.github/workflows/build.yml) workflow is the main process by which PRs and releases are built.
+The release process is kicked off whenever a branch name matches `release/v*` is pushed to [jpy-consortium/jpy](https://github.com/jpy-consortium/jpy).

jpyutil.py

@@ -214,19 +214,20 @@ def find_jvm_dll_file(java_home_dir=None, fail=False):
 
 
 def _get_jvm_lib_dirs(java_home_dir):
-    arch = 'amd64' if PYTHON_64BIT else 'i386'
+    machine = platform.machine()
+    arch_dir_name = 'aarch64' if machine == 'aarch64' else 'amd64' if PYTHON_64BIT else 'i386'
     return (os.path.join(java_home_dir, 'bin'),
             os.path.join(java_home_dir, 'bin', 'server'),
             os.path.join(java_home_dir, 'bin', 'client'),
-            os.path.join(java_home_dir, 'bin', arch),
-            os.path.join(java_home_dir, 'bin', arch, 'server'),
-            os.path.join(java_home_dir, 'bin', arch, 'client'),
+            os.path.join(java_home_dir, 'bin', arch_dir_name),
+            os.path.join(java_home_dir, 'bin', arch_dir_name, 'server'),
+            os.path.join(java_home_dir, 'bin', arch_dir_name, 'client'),
             os.path.join(java_home_dir, 'lib'),
             os.path.join(java_home_dir, 'lib', 'server'),
             os.path.join(java_home_dir, 'lib', 'client'),
-            os.path.join(java_home_dir, 'lib', arch),
-            os.path.join(java_home_dir, 'lib', arch, 'server'),
-            os.path.join(java_home_dir, 'lib', arch, 'client'),
+            os.path.join(java_home_dir, 'lib', arch_dir_name),
+            os.path.join(java_home_dir, 'lib', arch_dir_name, 'server'),
+            os.path.join(java_home_dir, 'lib', arch_dir_name, 'client'),
             )