Refresh One Notebook #11
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Refresh One Notebook | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| notebook: | |
| # --- refresh_notebooks_list.py: refresh list of notebooks currently available [START] --- | |
| # --- do not edit, auto generated part by `make refresh_notebooks_list` --- | |
| description: "Notebook file name only in: \n | |
| - Cifar10 \n | |
| - CifarInFhe \n | |
| - CifarInFheWithSmallerAccumulators \n | |
| - CifarQuantizationAwareTraining \n | |
| - ClassifierComparison \n | |
| - ClientServer \n | |
| - ConvolutionalNeuralNetwork \n | |
| - CreditScoring \n | |
| - DecisionTreeClassifier \n | |
| - DecisionTreeRegressor \n | |
| - Deployment \n | |
| - ExperimentPrivacyTreePaper \n | |
| - FromImageNetToCifar \n | |
| - FullyConnectedNeuralNetwork \n | |
| - FullyConnectedNeuralNetworkOnMNIST \n | |
| - GLMComparison \n | |
| - HealthCarePrediction \n | |
| - KaggleTitanic \n | |
| - LinearRegression \n | |
| - LinearSVR \n | |
| - LogisticRegression \n | |
| - PerrorImpactOnFMNIST \n | |
| - PoissonRegression \n | |
| - QGPT2Evaluate \n | |
| - QuantFrameworkExample \n | |
| - QuantizationAwareTraining \n | |
| - RegressorComparison \n | |
| - SentimentClassification \n | |
| - SVMClassifier \n | |
| - XGBClassifier \n | |
| - XGBRegressor \n | |
| " | |
| # --- refresh_notebooks_list.py: refresh list of notebooks currently available [END] --- | |
| required: true | |
| concurrency: | |
| group: "${{ github.ref }}-${{ github.event_name }}-${{ github.workflow }}" | |
| cancel-in-progress: false | |
| env: | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| # --- refresh_notebooks_list.py: refresh list of notebook paths currently available [START] --- | |
| # --- do not edit, auto generated part by `make refresh_notebooks_list` --- | |
| Cifar10: "use_case_examples/cifar/cifar_brevitas_with_model_splitting/Cifar10.ipynb" | |
| CifarInFhe: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarInFhe.ipynb" | |
| CifarInFheWithSmallerAccumulators: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarInFheWithSmallerAccumulators.ipynb" | |
| CifarQuantizationAwareTraining: "use_case_examples/cifar/cifar_brevitas_finetuning/CifarQuantizationAwareTraining.ipynb" | |
| ClassifierComparison: "docs/advanced_examples/ClassifierComparison.ipynb" | |
| ClientServer: "docs/advanced_examples/ClientServer.ipynb" | |
| ConvolutionalNeuralNetwork: "docs/advanced_examples/ConvolutionalNeuralNetwork.ipynb" | |
| CreditScoring: "use_case_examples/credit_scoring/CreditScoring.ipynb" | |
| DecisionTreeClassifier: "docs/advanced_examples/DecisionTreeClassifier.ipynb" | |
| DecisionTreeRegressor: "docs/advanced_examples/DecisionTreeRegressor.ipynb" | |
| Deployment: "docs/advanced_examples/Deployment.ipynb" | |
| ExperimentPrivacyTreePaper: "docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb" | |
| FromImageNetToCifar: "use_case_examples/cifar/cifar_brevitas_finetuning/FromImageNetToCifar.ipynb" | |
| FullyConnectedNeuralNetwork: "docs/advanced_examples/FullyConnectedNeuralNetwork.ipynb" | |
| FullyConnectedNeuralNetworkOnMNIST: "docs/advanced_examples/FullyConnectedNeuralNetworkOnMNIST.ipynb" | |
| GLMComparison: "docs/advanced_examples/GLMComparison.ipynb" | |
| HealthCarePrediction: "use_case_examples/disease_prediction/HealthCarePrediction.ipynb" | |
| KaggleTitanic: "use_case_examples/titanic/KaggleTitanic.ipynb" | |
| LinearRegression: "docs/advanced_examples/LinearRegression.ipynb" | |
| LinearSVR: "docs/advanced_examples/LinearSVR.ipynb" | |
| LogisticRegression: "docs/advanced_examples/LogisticRegression.ipynb" | |
| PerrorImpactOnFMNIST: "use_case_examples/cifar/cifar_brevitas_finetuning/PerrorImpactOnFMNIST.ipynb" | |
| PoissonRegression: "docs/advanced_examples/PoissonRegression.ipynb" | |
| QGPT2Evaluate: "use_case_examples/llm/QGPT2Evaluate.ipynb" | |
| QuantFrameworkExample: "use_case_examples/llm/QuantFrameworkExample.ipynb" | |
| QuantizationAwareTraining: "docs/advanced_examples/QuantizationAwareTraining.ipynb" | |
| RegressorComparison: "docs/advanced_examples/RegressorComparison.ipynb" | |
| SentimentClassification: "use_case_examples/sentiment_analysis_with_transformer/SentimentClassification.ipynb" | |
| SVMClassifier: "docs/advanced_examples/SVMClassifier.ipynb" | |
| XGBClassifier: "docs/advanced_examples/XGBClassifier.ipynb" | |
| XGBRegressor: "docs/advanced_examples/XGBRegressor.ipynb" | |
| # --- refresh_notebooks_list.py: refresh list of notebook paths currently available [END] --- | |
| jobs: | |
| start-runner-linux: | |
| name: Start EC2 runner | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| label-38: ${{ steps.start-ec2-runner-38.outputs.label }} | |
| ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Start EC2 runner python 38 | |
| id: start-ec2-runner-38 | |
| uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} | |
| ec2-image-id: ${{ secrets.AWS_EC2_AMI }} | |
| ec2-instance-type: "m6i.metal" | |
| subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} | |
| security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} | |
| refresh-one-notebook: | |
| needs: [start-runner-linux] | |
| runs-on: ${{ needs.start-runner-linux.outputs.label-38 }} | |
| # Run in a clean container | |
| container: | |
| image: ubuntu:20.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }} | |
| PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }} | |
| NOTEBOOK_NAME: ${{ github.event.inputs.notebook }} | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} | |
| steps: | |
| - name: Add masks | |
| run: | | |
| echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}" | |
| # Replace default archive.ubuntu.com from docker image with fr mirror | |
| # original archive showed performance issues and is farther away | |
| - name: Docker container related setup and git installation | |
| run: | | |
| TZ=Europe/Paris | |
| echo "TZ=${TZ}" >> "$GITHUB_ENV" | |
| ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone | |
| sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list | |
| apt update && apt install git git-lfs -y | |
| - name: Checkout Code | |
| uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 | |
| with: | |
| token: ${{ secrets.BOT_TOKEN }} | |
| lfs: true | |
| - name: 'Set up Python 3.8' | |
| uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 | |
| with: | |
| python-version: '3.8' | |
| - name: Install dependencies | |
| run: | | |
| ./script/make_utils/setup_os_deps.sh | |
| make setup_env | |
| - name: Download Sentiment classification data-sets | |
| if: ${{ env.NOTEBOOK_NAME == 'SentimentClassification' }} | |
| run: | | |
| source .venv/bin/activate | |
| cd ./use_case_examples/encrypted_sentiment_analysis && ./download_data.sh | |
| - name: Download Titanic data-sets | |
| if: ${{ env.NOTEBOOK_NAME == 'KaggleTitanic' }} | |
| run: | | |
| source .venv/bin/activate | |
| cd ./use_case_examples/titanic && ./download_data.sh | |
| - name: Refresh One Notebook | |
| run: | | |
| make jupyter_execute_one NOTEBOOK="${{ env[env.NOTEBOOK_NAME] }}" | |
| # Pull the latest changes if there are some | |
| - name: Pull latest changes | |
| run: | | |
| git pull -X theirs | |
| # If the target branch is main or a release branch, a Pull Request is opened for everyone to | |
| # review. | |
| - name: Open PR | |
| if: ${{ github.ref_name == 'main' || startsWith(github.ref_name , 'release/') }} | |
| uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 | |
| with: | |
| token: ${{ secrets.BOT_TOKEN }} | |
| commit-message: "chore: refresh ${{ github.event.inputs.notebook }} notebook" | |
| branch: "refresh-${{ github.event.inputs.notebook }}-notebook-for-${{ github.ref_name }}" | |
| base: "${{ github.ref_name }}" | |
| title: "Refresh ${{ github.event.inputs.notebook }} notebook for ${{ github.ref_name }}" | |
| body: "Automatic PR with notebook refresh of ${{ github.event.inputs.notebook }} \ | |
| for ${{ github.ref_name }}." | |
| add-paths: | | |
| docs/**/*.ipynb | |
| use_case_examples/**/*.ipynb | |
| # If the target branch is another branch, the current branch is automatically merged into it | |
| - name: Push changes into the current branch | |
| if: ${{ github.ref_name != 'main' && !(startsWith(github.ref_name , 'release/')) }} | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "chore: refresh ${{ github.event.inputs.notebook }} notebook" | |
| add_options: '-u' | |
| stop-runner-linux: | |
| name: Stop EC2 runner | |
| needs: [refresh-one-notebook, start-runner-linux] | |
| runs-on: ubuntu-20.04 | |
| if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Stop EC2 runner python 38 | |
| uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea | |
| if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }} | |
| with: | |
| github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} | |
| label: ${{ needs.start-runner-linux.outputs.label-38 }} | |
| ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }} | |
| mode: stop | |
| send-report: | |
| if: ${{ github.ref_name == 'main' || startsWith(github.ref_name , 'release/') }} | |
| needs: | |
| [ | |
| start-runner-linux, | |
| refresh-one-notebook, | |
| stop-runner-linux, | |
| ] | |
| # If the target branch is main or a release branch, a slack notification is sent | |
| name: Send Slack notification | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 | |
| - name: Prepare whole job status | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| env: | |
| NEEDS_JSON: ${{ toJSON(needs) }} | |
| run: | | |
| echo "${NEEDS_JSON}" > /tmp/needs_context.json | |
| JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \ | |
| --needs_context_json /tmp/needs_context.json) | |
| echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV" | |
| - name: Slack Notification | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8 | |
| env: | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }} | |
| SLACK_MESSAGE: "Refreshing ${{ github.event.inputs.notebook }} for ${{ github.ref_name }} | |
| finished with status ${{ env.JOB_STATUS || 'failure' }} (${{ env.ACTION_RUN_URL }})\n\ | |
| - start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\ | |
| - refresh-one-notebook: ${{ needs.refresh-one-notebook.result || 'Did not run.' }}\n\n\ | |
| - stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}" | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} |