Skip to content

Commit

Permalink
chore: add run use case example to GH workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
jfrery committed Apr 15, 2024
1 parent 718b7c2 commit d363066
Show file tree
Hide file tree
Showing 15 changed files with 293 additions and 216 deletions.
126 changes: 126 additions & 0 deletions .github/workflows/run_use_cases_examples.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
name: Run Use Case Examples
on:
push:
# workflow_dispatch:
# schedule:
# - cron: '0 0 * * MON' # Scheduled trigger every Monday at midnight

concurrency:
group: ${{ github.ref }}
cancel-in-progress: true

env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

jobs:
start-runner-linux:
name: Start EC2 runner
runs-on: ubuntu-20.04
outputs:
label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
steps:
- name: Checkout Code
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}

- name: Start EC2 runner python 38
id: start-ec2-runner-38
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
ec2-instance-type: "m6i.metal"
subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}

run-use-case-examples:
needs: [start-runner-linux]
runs-on: ${{ needs.start-runner-linux.outputs.label-38 }}
container:
image: ubuntu:20.04
defaults:
run:
shell: bash
steps:
- name: Checkout Code
uses: actions/checkout@v2

- name: Set up Environment
run: |
# Setup commands if any, for example, installing dependencies, etc.
apt-get update && apt-get install -y python3-venv make && apt install git git-lfs -y
- name: Run Use Case Examples Script
run: |
chmod +x ./script/make_utils/run_use_case_examples.sh
./script/make_utils/run_use_case_examples.sh
stop-runner-linux:
name: Stop EC2 runner
needs: [run-use-case-examples, start-runner-linux]
runs-on: ubuntu-20.04
if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}

- name: Stop EC2 runner python 38
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
with:
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-38 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
mode: stop

send-report:
if: ${{ always() }}
needs:
[
start-runner-linux,
run-use-case-examples,
stop-runner-linux,
]
name: Send Slack notification
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2 # Update to the latest stable version

- name: Prepare whole job status
if: ${{ always() }}
continue-on-error: true
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: |
echo "${NEEDS_JSON}" > /tmp/needs_context.json
JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \
--needs_context_json /tmp/needs_context.json)
echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV"
- name: Slack Notification
if: ${{ always() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8
env:
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }}
SLACK_MESSAGE: "Full run of use case examples finished with status ${{ env.JOB_STATUS || 'failure' }} \
(${{ env.ACTION_RUN_URL }})\n\
- start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\
- run-use-case-examples: ${{ needs.run-use-case-examples.result || 'Did not run.' }}\n\n\
- stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}"
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,7 @@ clean_sklearn_cache:

.PHONY: run_one_use_case_example # Run one use-case example (USE_CASE, eg use_case_examples/hybrid_model)
run_one_use_case_example:
./script/make_utils/run_use_case_examples.sh
USE_CASE=$(USE_CASE) ./script/make_utils/run_use_case_examples.sh

.PHONY: run_all_use_case_examples # Run all use-case examples
run_all_use_case_examples:
Expand Down
228 changes: 111 additions & 117 deletions script/make_utils/run_use_case_examples.sh
Original file line number Diff line number Diff line change
@@ -1,129 +1,123 @@
#!/usr/bin/env bash
set -e

DIR=$(dirname "$0")

# shellcheck disable=SC1090,SC1091
source "${DIR}/detect_docker.sh"

if isDocker; then
echo "Can not run in docker -> this script needs to install new virtualenvs"
exit 1
fi


CML_DIR=$(pwd)
USE_CASE_REL_DIR="use_case_examples"
USE_CASE_DIR="${CML_DIR}/${USE_CASE_REL_DIR}"

if [ ! -d "$USE_CASE_DIR" ]; then
echo "This script must be run in the Concrete ML source root where the '$USE_CASE_REL_DIR' directory is present"
exit 1
fi

echo "Refreshing notebooks with PIP installed Concrete ML"

# shellcheck disable=SC2143
if [[ $(git ls-files --others --exclude-standard | grep ${USE_CASE_REL_DIR}) ]]; then
echo "This script must be run in a clean clone of the Concrete ML repo"
echo "This directory has untracked files in ${USE_CASE_REL_DIR}"
echo "You can LIST all untracked files using: "
echo
# shellcheck disable=SC2028
echo " git ls-files --others --exclude-standard | grep ${USE_CASE_REL_DIR}"
echo
echo "You can REMOVE all untracked files using: "
echo
# shellcheck disable=SC2028
echo " git ls-files --others --exclude-standard | grep ${USE_CASE_REL_DIR} | xargs -0 -d '\n' --no-run-if-empty rm"
echo
exit 1
fi

if [[ -z "${USE_CASE}" ]]; then
# shellcheck disable=SC2207
LIST_OF_USE_CASES=($(find "$USE_CASE_DIR/" -mindepth 1 -maxdepth 2 -type d | grep -v checkpoints))
else
LIST_OF_USE_CASES=("${USE_CASE}")
if [ ! -d "${USE_CASE}" ]; then
echo "The use case specified to be executed, ${USE_CASE}, does not exist"
exit 1
fi
fi

if [ ! "$(docker images -q zamafhe/concrete-ml:latest 2> /dev/null)" ]; then
# BUILD THE DOCKER IMAGE
echo "Building docker image"
poetry build && mkdir -p pkg && cp dist/* pkg/ && make release_docker
docker tag concrete-ml-release:latest zamafhe/concrete-ml:latest
fi

# shellcheck disable=SC2068
for EXAMPLE in ${LIST_OF_USE_CASES[@]}
do
EXAMPLE_NAME=$(basename "${EXAMPLE}")

if [ -f "${EXAMPLE}/Makefile" ]; then
echo "*** Processing example ${EXAMPLE_NAME}"
CURRENT_DIR=$(pwd)
USE_CASE_DIR_NAME="use_case_examples"
USE_CASE_DIR="${CURRENT_DIR}/${USE_CASE_DIR_NAME}"

export USE_CASE_DIR # Required for the Makefile of the use case examples

check_directory_exists() {
if [ ! -d "$1" ]; then
echo "Error: Directory '${1}' not found."
exit 1
fi
}

check_clean_git_status() {
if git ls-files --others --exclude-standard | grep -q "$1"; then
echo "Error: The repository is not clean. Untracked files found in $1."
echo "List untracked files with: git ls-files --others --exclude-standard | grep $1"
echo "Remove untracked files with: git clean -fdx $1"
exit 1
fi
}

setup_virtualenv() {
local venv_path="/tmp/virtualenv_$1"
echo "Setting up virtual environment in $venv_path..."
python3 -m venv "$venv_path"
source "${venv_path}/bin/activate"
echo "Virtual environment activated."
}

install_concrete_ml() {
pip install -U pip setuptools wheel
if pip install -e .; then
echo "Concrete ML installation successful."
else
continue
echo "Failed to install Concrete ML."
return 1
fi
}

# Setup a new venv
VENV_PATH="/tmp/virtualenv_${EXAMPLE_NAME}"
if [ -d "$VENV_PATH" ]; then
echo " - VirtualEnv already exists, deleting the old one"
rm -rf "$VENV_PATH"
install_requirements() {
if [ -f "requirements.txt" ]; then
if pip install -r requirements.txt; then
echo "Requirements installed successfully."
else
echo "Failed to install requirements."
return 1
fi
fi
virtualenv -q "$VENV_PATH"
echo " - VirtualEnv created at $VENV_PATH"
# shellcheck disable=SC1090,SC1091
source "${VENV_PATH}/bin/activate"
# Install Concrete ML
set +e
cd "$CML_DIR"
pip install -e . &> "/tmp/log_cml_pip_${EXAMPLE_NAME}"
hresult=$?
if [ $hresult -ne 0 ]; then
echo "Could not install Concrete ML in the virtualenv, see /tmp/log_cml_pip_${EXAMPLE_NAME}"
rm -rf "$VENV_PATH"
continue
}

run_example() {
local example_dir=$1
local example_name=$(basename "$example_dir")

if [ ! -f "${example_dir}/Makefile" ]; then
echo "No Makefile found in $example_dir, skipping..."
return
fi
set -e
echo " - Concrete ML installed in $VENV_PATH"

# Install example requirements
cd "$EXAMPLE"
if [ -f "requirements.txt" ]; then
set +e
pip install -r requirements.txt &> "/tmp/log_reqs_${EXAMPLE_NAME}"
hresult=$?
set -e
if [ $hresult -ne 0 ]; then
echo "Could not install Concrete ML in the virtualenv, see /tmp/log_reqs_${EXAMPLE_NAME}"
rm -rf "$VENV_PATH"
continue
fi
echo " - Requirements installed in $VENV_PATH"
echo "*** Running example: $example_name ***"
setup_virtualenv "$example_name"
install_concrete_ml || return
install_requirements || return

echo "Running use case example using Makefile..."
make -C "$example_dir" run_example

local result="${PIPESTATUS[0]}"

if [ "$result" -ne 0 ]; then
echo "Failure in example $example_name."
failed_examples+=("$example_name")
else
echo "Successfully completed example $example_name."
success_examples+=("$example_name")
fi

set +e
# Strip colors from the error output before piping to the log files
# Swap stderr and stdout, all output of jupyter execution is in stderr
# The information about time spent running the notebook is in stdout
# The following will pipe the stderr to the regex so that it
# ends up in the log file.
# The timing shows in the terminal
USE_CASE_DIR=$USE_CASE_DIR make 3>&2 2>&1 1>&3- | perl -pe 's/\e([^\[\]]|\[.*?[a-zA-Z]|\].*?\a)//g' > "/tmp/log_${EXAMPLE_NAME}"

# Neet to check the result of execution of the make command (ignore the results
# of the other commands in the pipe)
hresult="${PIPESTATUS[0]}"
if [ "$hresult" -ne 0 ]; then
echo "Error while running example ${EXAMPLE_NAME} see /tmp/log_${EXAMPLE_NAME}"

deactivate
rm -rf "/tmp/virtualenv_$example_name"
}

print_summary() {
echo "Summary of execution results:"
echo "Successful examples: ${#success_examples[@]}"
for example in "${success_examples[@]}"; do
echo " - $example"
done
echo "Failed examples: ${#failed_examples[@]}"
for example in "${failed_examples[@]}"; do
echo " - $example"
done
}

main() {
check_directory_exists "$USE_CASE_DIR"
check_clean_git_status "$USE_CASE_DIR_NAME"

declare -a success_examples
declare -a failed_examples

local LIST_OF_USE_CASES=()
if [[ -z "${USE_CASE}" ]]; then
LIST_OF_USE_CASES=($(find "$USE_CASE_DIR/" -mindepth 1 -maxdepth 2 -type d | grep -v checkpoints | sort))
else
LIST_OF_USE_CASES=("${USE_CASE_DIR}/${USE_CASE}")
fi

for use_case in "${LIST_OF_USE_CASES[@]}"; do
run_example "$use_case"
done

print_summary

if [ ${#failed_examples[@]} -ne 0 ]; then
exit 1
fi
set -e
}

# Remove the virtualenv
rm -rf "$VENV_PATH"
done
main "$@"
7 changes: 5 additions & 2 deletions use_case_examples/cifar/cifar_brevitas_training/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8

EXAMPLE_NAME=cifar_brevitas_finetuning
EXAMPLE_NAME=cifar_brevitas_training
JUPYTER_RUN=jupyter nbconvert --to notebook --inplace --execute
TIME_NB="${USE_CASE_DIR}/time_notebook_execution.sh"

run_example: one
run_example: one two

one:
@python evaluate_one_example_fhe.py

two:
@python evaluate_torch_cml.py
Loading

0 comments on commit d363066

Please sign in to comment.