initial commit

.gitattributes

@@ -0,0 +1,10 @@
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.weight filter=lfs diff=lfs merge=lfs -text
+*.bias filter=lfs diff=lfs merge=lfs -text

.github/workflows/publish-to-test-pypi.yml

@@ -0,0 +1,34 @@
+name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+
+on:
+  push:
+    branches: ["main", "canary"]
+    tags:
+      - v*
+  pull_request:
+    branches: ["main", "canary"]
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@main
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: "latest"
+          activate-environment: tkml
+          python-version: ${{ matrix.python-version }}
+      - name: Install pypa/build
+        run: >-
+          python -m pip install build --user
+      - name: Build a binary wheel and a source tarball
+        run: >-
+          python -m build --sdist --wheel --outdir dist/ toolchain
+      - name: Test wheel
+        shell: bash -el {0}
+        run: |
+          python -m pip install --upgrade pip
+          pip install dist/*.whl
+          models=$(turnkey models location --quiet)
+          turnkey $models/selftest/linear.py

.github/workflows/test_azure.yml

@@ -0,0 +1,80 @@
+# This workflow will install Python dependencies and run a basic test to ensure
+# that the Cloud/Azure CLI is still working
+name: Test Azure CLI
+
+# TODO: enable these tests again once the repository is in the public domain 
+
+# on:
+#   push:
+#     branches: ["main"]
+#     paths:
+#       - toolchain/src/turnkeyml/cli/scale/**
+#   pull_request:
+#     branches: ["main"]
+#     paths:
+#       - toolchain/src/turnkeyml/cli/scale/**
+
+# In the meantime, allow manual execution
+on:
+  # Allows users to manually trigger the workflow using the GitHub UI
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build_and_test:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: "latest"
+          activate-environment: tkml
+          python-version: "3.8"
+      - name: Install dependencies
+        shell: bash -el {0}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y azure-cli
+          python -m pip install --upgrade pip
+          pip install -e .
+          pip install -r toolchain/src/turnkeyml/cli/cloud/requirements.txt
+      - name: Run tests
+        env:
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+          TURNKEY_AZURE_PUBLIC_KEY: ${{ secrets.AZURE_SSH_PUBLIC_KEY }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          TURNKEY_AZURE_PASSWORD: ${{ secrets.TURNKEY_AZURE_PASSWORD }}
+        shell: bash -el {0}
+        run: |
+          az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
+
+          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+          ssh-add - <<< "${{ secrets.AZURE_SSH_PRIVATE_KEY }}"
+
+          mkdir ~/.ssh
+          touch ~/.ssh/known_hosts
+
+          cd src/turnkeyml/cli/cloud
+
+          python control.py create setup selftest info stop --name github-test-azure-single --hardware cpu-small
+
+          python control.py create setup selftest info stop --cluster --name github-test-azure-cluster --size 2 --hardware cpu-small
+          python control.py start run --cluster --name github-test-azure-cluster --size 2 --hardware cpu-small --input-files onnxmodelzoo/toolchain/models/selftest/*.py
+  clean_up:
+    needs: build_and_test
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install az cli
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y azure-cli
+      - name: Delete resource group
+        run: |
+          az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
+          az group delete --name github-test-azure-single-rg --yes
+          az group delete --name github-test-azure-cluster-rg --yes

.github/workflows/test_build_api.yml

@@ -0,0 +1,81 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Lint and Test Build API
+
+on:
+  push:
+    branches: ["main", "canary"]
+  pull_request:
+    branches: ["main", "canary"]
+
+permissions:
+  contents: read
+
+jobs:
+  build-the-build-api:
+    env:
+      TURNKEY_DEBUG: True
+      TURNKEY_TRACEBACK: True
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.10"]
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Miniconda with 64-bit Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: "latest"
+          activate-environment: tkml
+          python-version: ${{ matrix.python-version }}
+          architecture: "x64"
+      - name: Install dependencies
+        shell: bash -el {0}
+        run: |
+          python -m pip install --upgrade pip
+          conda install pylint
+          pip install -e toolchain[tensorflow] --no-cache-dir
+          pip install transformers
+          python -m pip check
+      - name: Lint with PyLint
+        shell: bash -el {0}
+        run: |
+          pylint toolchain/src/turnkeyml/build --rcfile toolchain/.pylintrc
+          pylint toolchain/examples/build_api --rcfile toolchain/.pylintrc
+      - name: Test with unittest
+        shell: bash -el {0}
+        run: |
+
+          # build api examples
+          # Note: we clear the default cache location prior to each example run
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/hello_torch_world.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/hello_keras_world.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/hello_onnx_world.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/randomforest.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/xgbclassifier.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/lgbmclassifier.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/kneighborsclassifier.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/build_name.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/cache_dir.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/no_monitor.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/rebuild_always.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/rebuild_never.py
+          rm -rf ~/.cache/turnkey_test_cache
+          python toolchain/examples/build_api/sequence.py
+
+          # build api tests
+          python toolchain/test/build_model.py

.github/workflows/test_gpu_turnkey.yml

@@ -0,0 +1,87 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Test TurnkeyML on Azure GPUs
+
+on:
+  # Allows users to manually trigger the workflow using the GitHub UI
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  start_vm:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install az cli
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y azure-cli
+      - name: Turn on Azure VM
+        run: |
+          az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
+          az vm wait --name morecuda --resource-group morecuda_group --custom "instanceView.statuses[?code=='PowerState/deallocated']" --interval 30
+          az vm start --name morecuda --resource-group morecuda_group
+          az vm wait --created --name morecuda --resource-group morecuda_group
+          az vm run-command invoke --name morecuda --resource-group morecuda_group --command-id RunShellScript --scripts 'export RUNNER_ALLOW_RUNASROOT="1" && export HOME="/home/azureuser" && bash /home/azureuser/actions-runner/run.sh &'
+  build_and_test:
+    needs: start_vm
+    timeout-minutes: 10
+    if: ${{ needs.start_vm.result == 'success' }}
+    runs-on: self-hosted
+    env:
+      TURNKEY_DEBUG: True
+      TURNKEY_TRACEBACK: True
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: "latest"
+          activate-environment: tkml
+          python-version: ${{ matrix.python-version }}
+      - name: Install Docker
+        shell: bash -el {0}
+        run: |
+          sudo mkdir -p /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
+          sudo chmod a+r /etc/apt/keyrings/docker.gpg
+          echo \
+          "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+          "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+          sudo chmod 666 /var/run/docker.sock
+      - name: Install dependencies
+        shell: bash -el {0}
+        run: |
+          python -m pip install --upgrade pip
+          conda install pylint
+          pip install -e toolchain
+          pip install transformers
+          python -m pip check
+      - name: Test with unittest
+        shell: bash -el {0}
+        run: |
+          # E2E tests
+          python toolchain/test/gpu.py
+  deallocate_vm:
+    needs: build_and_test
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install az cli
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y azure-cli
+      - name: Deallocate Azure VM
+        run: |
+          az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
+          az vm deallocate --name morecuda --resource-group morecuda_group