Skip to content

Refresh One Notebook #22

Refresh One Notebook

Refresh One Notebook #22

name: Refresh One Notebook
on:
workflow_dispatch:
inputs:
notebook:
# --- refresh_notebooks_list.py: refresh list of notebooks currently available [START] ---
# --- do not edit, auto generated part by `make refresh_notebooks_list` ---
description: "Notebook file name only in: \n
- Cifar10 \n
- CifarInFhe \n
- CifarInFheWithSmallerAccumulators \n
- CifarQuantizationAwareTraining \n
- ClassifierComparison \n
- ClientServer \n
- ConvolutionalNeuralNetwork \n
- CreditScoring \n
- DecisionTreeClassifier \n
- DecisionTreeRegressor \n
- Deployment \n
- ExperimentPrivacyTreePaper \n
- FromImageNetToCifar \n
- FullyConnectedNeuralNetwork \n
- FullyConnectedNeuralNetworkOnMNIST \n
- GLMComparison \n
- HealthCarePrediction \n
- KaggleTitanic \n
- KNearestNeighbors \n
- LinearRegression \n
- LinearSVR \n
- LogisticRegression \n
- LogisticRegressionTraining \n
- PerrorImpactOnFMNIST \n
- PoissonRegression \n
- QGPT2Evaluate \n
- QuantFrameworkExample \n
- QuantizationAwareTraining \n
- RegressorComparison \n
- SentimentClassification \n
- SVMClassifier \n
- XGBClassifier \n
- XGBRegressor \n
"
# --- refresh_notebooks_list.py: refresh list of notebooks currently available [END] ---
required: true
concurrency:
group: "${{ github.ref }}-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.inputs.notebook }}"
cancel-in-progress: false
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
# --- refresh_notebooks_list.py: refresh list of notebook paths currently available [START] ---
# --- do not edit, auto generated part by `make refresh_notebooks_list` ---
Cifar10: "use_case_examples/cifar/cifar_brevitas_with_model_splitting/Cifar10.ipynb"
CifarInFhe: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarInFhe.ipynb"
CifarInFheWithSmallerAccumulators: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarInFheWithSmallerAccumulators.ipynb"
CifarQuantizationAwareTraining: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarQuantizationAwareTraining.ipynb"
ClassifierComparison: "docs/advanced_examples/ClassifierComparison.ipynb"
ClientServer: "docs/advanced_examples/ClientServer.ipynb"
ConvolutionalNeuralNetwork: "docs/advanced_examples/ConvolutionalNeuralNetwork.ipynb"
CreditScoring: "use_case_examples/credit_scoring/CreditScoring.ipynb"
DecisionTreeClassifier: "docs/advanced_examples/DecisionTreeClassifier.ipynb"
DecisionTreeRegressor: "docs/advanced_examples/DecisionTreeRegressor.ipynb"
Deployment: "docs/advanced_examples/Deployment.ipynb"
ExperimentPrivacyTreePaper: "docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb"
FromImageNetToCifar: "use_case_examples/cifar/cifar_brevitas_finetuning/FromImageNetToCifar.ipynb"
FullyConnectedNeuralNetwork: "docs/advanced_examples/FullyConnectedNeuralNetwork.ipynb"
FullyConnectedNeuralNetworkOnMNIST: "docs/advanced_examples/FullyConnectedNeuralNetworkOnMNIST.ipynb"
GLMComparison: "docs/advanced_examples/GLMComparison.ipynb"
HealthCarePrediction: "use_case_examples/disease_prediction/HealthCarePrediction.ipynb"
KaggleTitanic: "use_case_examples/titanic/KaggleTitanic.ipynb"
KNearestNeighbors: "docs/advanced_examples/KNearestNeighbors.ipynb"
LinearRegression: "docs/advanced_examples/LinearRegression.ipynb"
LinearSVR: "docs/advanced_examples/LinearSVR.ipynb"
LogisticRegression: "docs/advanced_examples/LogisticRegression.ipynb"
LogisticRegressionTraining: "docs/advanced_examples/LogisticRegressionTraining.ipynb"
PerrorImpactOnFMNIST: "use_case_examples/cifar/cifar_brevitas_finetuning/PerrorImpactOnFMNIST.ipynb"
PoissonRegression: "docs/advanced_examples/PoissonRegression.ipynb"
QGPT2Evaluate: "use_case_examples/llm/QGPT2Evaluate.ipynb"
QuantFrameworkExample: "use_case_examples/llm/QuantFrameworkExample.ipynb"
QuantizationAwareTraining: "docs/advanced_examples/QuantizationAwareTraining.ipynb"
RegressorComparison: "docs/advanced_examples/RegressorComparison.ipynb"
SentimentClassification: "use_case_examples/sentiment_analysis_with_transformer/SentimentClassification.ipynb"
SVMClassifier: "docs/advanced_examples/SVMClassifier.ipynb"
XGBClassifier: "docs/advanced_examples/XGBClassifier.ipynb"
XGBRegressor: "docs/advanced_examples/XGBRegressor.ipynb"
# --- refresh_notebooks_list.py: refresh list of notebook paths currently available [END] ---
jobs:
start-runner-linux:
name: Start EC2 runner
runs-on: ubuntu-20.04
outputs:
label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
steps:
- name: Checkout Code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Start EC2 runner python 38
id: start-ec2-runner-38
uses: machulav/ec2-github-runner@2c4d1dcf2c54673ed3bfd194c4b6919ed396a209
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
ec2-instance-type: "m6i.metal"
subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
refresh-one-notebook:
needs: [start-runner-linux]
runs-on: ${{ needs.start-runner-linux.outputs.label-38 }}
# Run in a clean container
container:
image: ubuntu:20.04
defaults:
run:
shell: bash
env:
PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }}
PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }}
NOTEBOOK_NAME: ${{ github.event.inputs.notebook }}
KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}
steps:
- name: Add masks
run: |
echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}"
echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}"
# Replace default archive.ubuntu.com from docker image with fr mirror
# original archive showed performance issues and is farther away
- name: Docker container related setup and git installation
run: |
TZ=Europe/Paris
echo "TZ=${TZ}" >> "$GITHUB_ENV"
ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
apt update && apt install git git-lfs -y
- name: Checkout Code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
token: ${{ secrets.BOT_TOKEN }}
lfs: true
- name: 'Set up Python 3.8'
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
with:
python-version: '3.8'
- name: Install dependencies
run: |
./script/make_utils/setup_os_deps.sh
make setup_env
- name: Download Sentiment classification data-sets
if: ${{ env.NOTEBOOK_NAME == 'SentimentClassification' }}
run: |
source .venv/bin/activate
cd ./use_case_examples/encrypted_sentiment_analysis && ./download_data.sh
- name: Download Titanic data-sets
if: ${{ env.NOTEBOOK_NAME == 'KaggleTitanic' }}
run: |
source .venv/bin/activate
cd ./use_case_examples/titanic && ./download_data.sh
- name: Refresh ${{ github.event.inputs.notebook }}
run: |
make jupyter_execute_one NOTEBOOK="${{ env[env.NOTEBOOK_NAME] }}"
# Pull the latest changes if there are some
- name: Pull latest changes
run: |
git pull -X theirs
# If the target branch is main or a release branch, a Pull Request is opened for everyone to
# review.
- name: Open PR
if: ${{ github.ref_name == 'main' || startsWith(github.ref_name , 'release/') }}
uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666
with:
token: ${{ secrets.BOT_TOKEN }}
commit-message: "chore: refresh ${{ github.event.inputs.notebook }} notebook"
branch: "refresh-${{ github.event.inputs.notebook }}-notebook-for-${{ github.ref_name }}"
base: "${{ github.ref_name }}"
title: "Refresh ${{ github.event.inputs.notebook }} notebook for ${{ github.ref_name }}"
body: "Automatic PR with notebook refresh of ${{ github.event.inputs.notebook }} \
for ${{ github.ref_name }}."
add-paths: |
docs/**/*.ipynb
use_case_examples/**/*.ipynb
# If the target branch is another branch, the current branch is automatically merged into it
- name: Push changes into the current branch
if: ${{ github.ref_name != 'main' && !(startsWith(github.ref_name , 'release/')) }}
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "chore: refresh ${{ github.event.inputs.notebook }} notebook"
add_options: '-u'
stop-runner-linux:
name: Stop EC2 runner
needs: [refresh-one-notebook, start-runner-linux]
runs-on: ubuntu-20.04
if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Stop EC2 runner python 38
uses: machulav/ec2-github-runner@2c4d1dcf2c54673ed3bfd194c4b6919ed396a209
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
with:
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-38 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
mode: stop
send-report:
if: ${{ github.ref_name == 'main' || startsWith(github.ref_name , 'release/') }}
needs:
[
start-runner-linux,
refresh-one-notebook,
stop-runner-linux,
]
# If the target branch is main or a release branch, a slack notification is sent
name: Send Slack notification
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Prepare whole job status
if: ${{ always() }}
continue-on-error: true
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: |
echo "${NEEDS_JSON}" > /tmp/needs_context.json
JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \
--needs_context_json /tmp/needs_context.json)
echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV"
- name: Slack Notification
continue-on-error: true
uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8
env:
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }}
SLACK_MESSAGE: "Refreshing ${{ github.event.inputs.notebook }} for ${{ github.ref_name }}
finished with status ${{ env.JOB_STATUS || 'failure' }} (${{ env.ACTION_RUN_URL }})\n\
- start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\
- refresh-one-notebook: ${{ needs.refresh-one-notebook.result || 'Did not run.' }}\n\n\
- stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}"
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}