From 78eca93903715d54b23316ba38f66790ea688c23 Mon Sep 17 00:00:00 2001 From: Yiyao Wei Date: Tue, 10 Aug 2021 14:07:29 +0200 Subject: [PATCH] feat: Add Fine tune option + refactoring (#7) * add fine tune option * test fine tune in CI * fix typo * adjust args * Update duckling url * Update README * Update duckling url * Get rasa version from rasa-demo repo * Split `test_args` to `test_nlu_args` and `test_core_args` * Revert "Split `test_args` to `test_nlu_args` and `test_core_args`" This reverts commit e765f43442f7df8662417e0774db9eb884303d60. * Split rasa test * Add train type * Split arguments for rasa test nlu and rasa test core * More comments * Change test type to all --- .github/test_files/config_rasa_demo.yml | 2 +- .github/workflows/ci.yml | 27 ++++++++++-- .github/workflows/ci_summary.yml | 20 ++++++++- README.md | 33 +++++++++++++- action.yml | 58 ++++++++++++++++++++++--- 5 files changed, 124 insertions(+), 16 deletions(-) diff --git a/.github/test_files/config_rasa_demo.yml b/.github/test_files/config_rasa_demo.yml index 09f976d..89c59d0 100644 --- a/.github/test_files/config_rasa_demo.yml +++ b/.github/test_files/config_rasa_demo.yml @@ -14,7 +14,7 @@ pipeline: epochs: 5 ranking_length: 5 - name: DucklingEntityExtractor - url: http://localhost:8000 + url: http://duckling.rasa.com:8000 dimensions: - email - number diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdff094..3a8fd8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: rasa_version: ${{ env.RASA_IMAGE_TAG }} workspace: ${{ github.workspace }}/rasa-project test_type: all + fine_tune: 'true' github_token: ${{ secrets.GITHUB_TOKEN }} configuration_name: 'rasa-init' publish_summary: 'false' @@ -79,6 +80,7 @@ jobs: rasa_version: ${{ env.RASA_IMAGE_TAG }} workspace: ${{ github.workspace }}/rasa-x-demo test_type: all + fine_tune: 'true' github_token: ${{ secrets.GITHUB_TOKEN }} model: github_workspace/test_model.tar.gz configuration: github_workspace/test_files/config_rasa_x_demo.yml @@ -117,16 +119,21 @@ jobs: run: | cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml + - name: Set Rasa Version + run: | + RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt) + echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV + - id: action name: Rasa Test & Train - Rasa Demo uses: ./ with: - rasa_version: '2.0.0-full' + rasa_version: ${{ env.RASA_VERSION }}-full configuration_name: 'Rasa Demo - NLU' workspace: ${{ github.workspace }}/rasa-demo test_type: nlu github_token: ${{ secrets.GITHUB_TOKEN }} - test_args: '-f 2' + test_nlu_args: '-f 2' # Use cross validation cross_validation: 'true' # Compare results to the test report @@ -152,11 +159,23 @@ jobs: run: | cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml + - name: Set Rasa Version + run: | + RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt) + echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV + - id: action name: Rasa Test & Train - Rasa Demo uses: ./ with: - rasa_version: '2.0.0-full' - configuration_name: 'Rasa Demo' + rasa_version: ${{ env.RASA_VERSION }}-full + configuration_name: 'Rasa Demo - ALL' workspace: ${{ github.workspace }}/rasa-demo + test_type: all + # The arguments algin with the CI in rasa-demo + # https://github.com/RasaHQ/rasa-demo/blob/main/.github/workflows/continuous_integration.yml#L199-L207 + test_core_args: >- + --stories tests/test_conversations.yml + --fail-on-prediction-errors github_token: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/ci_summary.yml b/.github/workflows/ci_summary.yml index 641859f..b91864c 100644 --- a/.github/workflows/ci_summary.yml +++ b/.github/workflows/ci_summary.yml @@ -25,13 +25,21 @@ jobs: run: | cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml + - name: Set Rasa Version + run: | + RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt) + echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV + - id: action name: Rasa Test & Train - Rasa Demo uses: ./ with: - rasa_version: '2.0.0-full' + rasa_version: ${{ env.RASA_VERSION }}-full configuration_name: 'Rasa Demo - Configuration 1' workspace: ${{ github.workspace }}/rasa-demo + test_core_args: >- + --stories tests/test_conversations.yml + --fail-on-prediction-errors publish_summary: 'false' - uses: actions/upload-artifact@v2 @@ -62,13 +70,21 @@ jobs: run: | cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml + - name: Set Rasa Version + run: | + RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt) + echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV + - id: action name: Rasa Test & Train - Rasa Demo uses: ./ with: - rasa_version: '2.0.0-full' + rasa_version: ${{ env.RASA_VERSION }}-full configuration_name: 'Rasa Demo - Configuration 2' workspace: ${{ github.workspace }}/rasa-demo + test_core_args: >- + --stories tests/test_conversations.yml + --fail-on-prediction-errors github_token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/upload-artifact@v2 diff --git a/README.md b/README.md index 7fcaf39..c1c4cf3 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,13 @@ jobs: | `rasa_train` | Run `rasa train` | `true` | | `rasa_test` | Run `rasa test` | `true` | | `data_validate` | Validates domain and data files to check for possible mistakes | `true` | +| `fine_tune` | Fine-tune an existing model with new training dataset | `false` | | `workspace` | The root directory containing your Rasa Open Source project | `${{ github.workspace }}` | +| `train_type` | The types of training (available types: `core`/`nlu`/`all`) | `all` | | `train_args` | Additional arguments passed to the `rasa train` command | `none` | -| `test_args` | Additional arguments passed to the `rasa test` command | `none` | | `test_type` | The types of tests to run (available types: `core`/`nlu`/`all`) | `all` | +| `test_nlu_args` | Additional arguments passed to the `rasa test nlu` command | `none` | +| `test_core_args` | Additional arguments passed to the `rasa test core` command | `none` | | `publish_summary` | Publish tests summary as a PR comment | `true` | | `github_token` | GitHub Token - required to add a comment with summary | `none` | | `configuration` | Model configuration file | `config.yml` | @@ -169,6 +172,32 @@ jobs: # ... ``` +### Fine-tune the existing model + +You can fine-tune an existing model with new training dataset. Please see [incremental training](https://rasa.com/docs/rasa/command-line-interface#incremental-training) for more details. + +```yaml +jobs: + train_and_test: + # ... + steps: + # ... + - name: Train and Test Rasa Demo + uses: RasaHQ/rasa-train-test-gha@main + with: + # List of available tags: https://hub.docker.com/r/rasa/rasa/tags + rasa_version: '2.0.0-full' + # In order to add a PR comment with summary + # a GH Token has to be pass to the GH action + github_token: ${{ secrets.GITHUB_TOKEN }} + # By default, the number of epoches is defined in model configuration + # You can shorten the epoches by using the --epoch-fraction flag + fine_tune: 'true' + train_args: >- + --epoch-fraction 0.5 + # ... +``` + ### Cross-validate NLU model ```yaml @@ -189,7 +218,7 @@ jobs: cross_validation: 'true' # Number of cross validation folds (cross validation only) # All available arguments: rasa test nlu --help - test_args: '--folds 3' + test_nlu_args: '--folds 3' test_type: 'nlu' # ... ``` diff --git a/action.yml b/action.yml index 24eceaa..b36e6e5 100644 --- a/action.yml +++ b/action.yml @@ -17,6 +17,10 @@ inputs: description: 'Run rasa test' required: true default: 'true' + fine_tune: + description: 'Fine-tune model' + required: false + default: 'false' data_validate: description: 'Validates domain and data files to check for possible mistakes' required: true @@ -25,12 +29,20 @@ inputs: description: 'The root directory' required: false default: '${{ github.workspace }}' + train_type: + description: 'The types of training (available types: core/nlu/all)' + required: false + default: 'all' train_args: description: 'Additional arguments passed to the rasa train command' required: false default: '' - test_args: - description: 'Additional arguments passed to the rasa test command' + test_nlu_args: + description: 'Additional arguments passed to the rasa test nlu command' + required: false + default: '' + test_core_args: + description: 'Additional arguments passed to the rasa test core command' required: false default: '' test_type: @@ -97,16 +109,33 @@ runs: mkdir -p ${{ inputs.workspace }}/${{ inputs.result_directory }} chmod 777 ${{ inputs.workspace }}/${{ inputs.result_directory }} + case "${{ inputs.train_type }}" in + all) + echo TRAIN_TYPE="" >> $GITHUB_ENV + ;; + nlu) + echo TRAIN_TYPE=nlu >> $GITHUB_ENV + echo DEFAULT_ARGS="--config /app/${{ inputs.configuration }}" >> $GITHUB_ENV + ;; + core) + echo TRAIN_TYPE=core >> $GITHUB_ENV + ;; + *) + echo "::error::Unknown training type: ${{ inputs.train_type }} (use: core/nlu/all)" + exit 1 + esac + case "${{ inputs.test_type }}" in all) - echo TEST_TYPE="" >> $GITHUB_ENV + echo TEST_NLU="true" >> $GITHUB_ENV + echo TEST_CORE="true" >> $GITHUB_ENV ;; nlu) - echo TEST_TYPE=nlu >> $GITHUB_ENV + echo TEST_NLU="true" >> $GITHUB_ENV echo DEFAULT_ARGS="--config /app/${{ inputs.configuration }}" >> $GITHUB_ENV ;; core) - echo TEST_TYPE=core >> $GITHUB_ENV + echo TEST_CORE="true" >> $GITHUB_ENV ;; *) echo "::error::Unknown test type: ${{ inputs.test_type }} (use: core/nlu/all)" @@ -179,6 +208,16 @@ runs: echo "::warning::Data validation is disabled. To turn on data validation set the data_validate parameter to 'true'." fi + - name: Fine Tune + shell: bash + run: | + if [[ "${{ inputs.fine_tune }}" == "true" ]]; then + echo "Fine tuning model is enabled." + echo FINE_TUNE="--finetune" >> $GITHUB_ENV + else + echo "::warning::Fine tune is disabled. To turn on fine-tune, set the fine_tune parameter to 'true'." + fi + - name: Run the rasa train command shell: bash run: | @@ -189,7 +228,7 @@ runs: elif [[ "${{ inputs.rasa_train }}" == "true" ]]; then echo "Run the rasa train" echo "" - docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} train ${{ env.TEST_TYPE }} ${{ env.DEFAULT_ARGS }} ${{ inputs.train_args }} + docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} train ${{ env.TRAIN_TYPE }} ${{ env.DEFAULT_ARGS }} ${{ env.FINE_TUNE }} ${{ inputs.train_args }} else echo "::warning::rasa train is disabled. To turn on the rasa train set the rasa_train parameter to 'true'." fi @@ -200,7 +239,12 @@ runs: if [[ "${{ inputs.rasa_test }}" == "true" ]]; then echo "Run the rasa test" echo "" - docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test ${{ env.TEST_TYPE }} ${{ env.MODEL_ARGS }} ${{ env.CROSS_VALIDATION_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_args }} + if [[ "${{ env.TEST_NLU }}" == "true" ]]; then + docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test nlu ${{ env.MODEL_ARGS }} ${{ env.CROSS_VALIDATION_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_nlu_args }} + fi + if [[ "${{ env.TEST_CORE }}" == "true" ]]; then + docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test core ${{ env.MODEL_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_core_args }} + fi else echo "::warning::rasa test is disabled. To turn on the rasa train set the rasa_test parameter to 'true'." fi