feat: Add Fine tune option + refactoring (#7)

* add fine tune option * test fine tune in CI * fix typo * adjust args * Update duckling url * Update README * Update duckling url * Get rasa version from rasa-demo repo * Split `test_args` to `test_nlu_args` and `test_core_args` * Revert "Split `test_args` to `test_nlu_args` and `test_core_args`" This reverts commit e765f43. * Split rasa test * Add train type * Split arguments for rasa test nlu and rasa test core * More comments * Change test type to all
RasaHQ · Aug 10, 2021 · 78eca93 · 78eca93
1 parent 19a99c1
commit 78eca93
Show file tree

Hide file tree

Showing 5 changed files with 124 additions and 16 deletions.
diff --git a/.github/test_files/config_rasa_demo.yml b/.github/test_files/config_rasa_demo.yml
@@ -14,7 +14,7 @@ pipeline:
  epochs: 5
  ranking_length: 5
 - name: DucklingEntityExtractor
- url: http://localhost:8000
+ url: http://duckling.rasa.com:8000
  dimensions:
  - email
  - number

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -37,6 +37,7 @@ jobs:
  rasa_version: ${{ env.RASA_IMAGE_TAG }}
  workspace: ${{ github.workspace }}/rasa-project
  test_type: all
+ fine_tune: 'true'
  github_token: ${{ secrets.GITHUB_TOKEN }}
  configuration_name: 'rasa-init'
  publish_summary: 'false'
@@ -79,6 +80,7 @@ jobs:
  rasa_version: ${{ env.RASA_IMAGE_TAG }}
  workspace: ${{ github.workspace }}/rasa-x-demo
  test_type: all
+ fine_tune: 'true'
  github_token: ${{ secrets.GITHUB_TOKEN }}
  model: github_workspace/test_model.tar.gz
  configuration: github_workspace/test_files/config_rasa_x_demo.yml
@@ -117,16 +119,21 @@ jobs:
  run: |
  cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml
 
+ - name: Set Rasa Version
+ run: |
+ RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt)
+ echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV
+
  - id: action
  name: Rasa Test & Train - Rasa Demo
  uses: ./
  with:
- rasa_version: '2.0.0-full'
+ rasa_version: ${{ env.RASA_VERSION }}-full
  configuration_name: 'Rasa Demo - NLU'
  workspace: ${{ github.workspace }}/rasa-demo
  test_type: nlu
  github_token: ${{ secrets.GITHUB_TOKEN }}
- test_args: '-f 2'
+ test_nlu_args: '-f 2'
  # Use cross validation
  cross_validation: 'true'
  # Compare results to the test report
@@ -152,11 +159,23 @@ jobs:
  run: |
  cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml
 
+ - name: Set Rasa Version
+ run: |
+ RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt)
+ echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV
+
  - id: action
  name: Rasa Test & Train - Rasa Demo
  uses: ./
  with:
- rasa_version: '2.0.0-full'
- configuration_name: 'Rasa Demo'
+ rasa_version: ${{ env.RASA_VERSION }}-full
+ configuration_name: 'Rasa Demo - ALL'
  workspace: ${{ github.workspace }}/rasa-demo
+ test_type: all
+ # The arguments algin with the CI in rasa-demo
+ # https://github.com/RasaHQ/rasa-demo/blob/main/.github/workflows/continuous_integration.yml#L199-L207
+ test_core_args: >-
+ --stories tests/test_conversations.yml
+ --fail-on-prediction-errors
  github_token: ${{ secrets.GITHUB_TOKEN }}
+
diff --git a/.github/workflows/ci_summary.yml b/.github/workflows/ci_summary.yml
@@ -25,13 +25,21 @@ jobs:
  run: |
  cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml
 
+ - name: Set Rasa Version
+ run: |
+ RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt)
+ echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV
+
  - id: action
  name: Rasa Test & Train - Rasa Demo
  uses: ./
  with:
- rasa_version: '2.0.0-full'
+ rasa_version: ${{ env.RASA_VERSION }}-full
  configuration_name: 'Rasa Demo - Configuration 1'
  workspace: ${{ github.workspace }}/rasa-demo
+ test_core_args: >-
+ --stories tests/test_conversations.yml
+ --fail-on-prediction-errors
  publish_summary: 'false'
 
  - uses: actions/upload-artifact@v2
@@ -62,13 +70,21 @@ jobs:
  run: |
  cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml
 
+ - name: Set Rasa Version
+ run: |
+ RASA_VERSION=$(sed -ne 's/^rasa==\(.*\)#.*$/\1/p' rasa-demo/requirements.txt)
+ echo RASA_VERSION=$RASA_VERSION >> $GITHUB_ENV
+
  - id: action
  name: Rasa Test & Train - Rasa Demo
  uses: ./
  with:
- rasa_version: '2.0.0-full'
+ rasa_version: ${{ env.RASA_VERSION }}-full
  configuration_name: 'Rasa Demo - Configuration 2'
  workspace: ${{ github.workspace }}/rasa-demo
+ test_core_args: >-
+ --stories tests/test_conversations.yml
+ --fail-on-prediction-errors
  github_token: ${{ secrets.GITHUB_TOKEN }}
 
  - uses: actions/upload-artifact@v2

diff --git a/README.md b/README.md
@@ -30,10 +30,13 @@ jobs:
 | `rasa_train` | Run `rasa train` | `true` |
 | `rasa_test` | Run `rasa test` | `true` |
 | `data_validate` | Validates domain and data files to check for possible mistakes | `true` |
+| `fine_tune` | Fine-tune an existing model with new training dataset | `false` |
 | `workspace` | The root directory containing your Rasa Open Source project | `${{ github.workspace }}` |
+| `train_type` | The types of training (available types: `core`/`nlu`/`all`) | `all` |
 | `train_args` | Additional arguments passed to the `rasa train` command | `none` |
-| `test_args` | Additional arguments passed to the `rasa test` command | `none` |
 | `test_type` | The types of tests to run (available types: `core`/`nlu`/`all`) | `all` |
+| `test_nlu_args` | Additional arguments passed to the `rasa test nlu` command | `none` |
+| `test_core_args` | Additional arguments passed to the `rasa test core` command | `none` |
 | `publish_summary` | Publish tests summary as a PR comment | `true` |
 | `github_token` | GitHub Token - required to add a comment with summary | `none` |
 | `configuration` | Model configuration file | `config.yml` |
@@ -169,6 +172,32 @@ jobs:
  # ...
 ```
 
+### Fine-tune the existing model
+
+You can fine-tune an existing model with new training dataset. Please see [incremental training](https://rasa.com/docs/rasa/command-line-interface#incremental-training) for more details.
+
+```yaml
+jobs:
+ train_and_test:
+ # ...
+ steps:
+ # ...
+ - name: Train and Test Rasa Demo
+ uses: RasaHQ/rasa-train-test-gha@main
+ with:
+ # List of available tags: https://hub.docker.com/r/rasa/rasa/tags
+ rasa_version: '2.0.0-full'
+ # In order to add a PR comment with summary
+ # a GH Token has to be pass to the GH action
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ # By default, the number of epoches is defined in model configuration
+ # You can shorten the epoches by using the --epoch-fraction flag
+ fine_tune: 'true'
+ train_args: >-
+ --epoch-fraction 0.5
+ # ...
+```
+
 ### Cross-validate NLU model
 
 ```yaml
@@ -189,7 +218,7 @@ jobs:
  cross_validation: 'true'
  # Number of cross validation folds (cross validation only)
  # All available arguments: rasa test nlu --help
- test_args: '--folds 3'
+ test_nlu_args: '--folds 3'
  test_type: 'nlu'
  # ...
 ```

diff --git a/action.yml b/action.yml
@@ -17,6 +17,10 @@ inputs:
  description: 'Run rasa test'
  required: true
  default: 'true'
+ fine_tune:
+ description: 'Fine-tune model'
+ required: false
+ default: 'false'
  data_validate:
  description: 'Validates domain and data files to check for possible mistakes'
  required: true
@@ -25,12 +29,20 @@ inputs:
  description: 'The root directory'
  required: false
  default: '${{ github.workspace }}'
+ train_type:
+ description: 'The types of training (available types: core/nlu/all)'
+ required: false
+ default: 'all'
  train_args:
  description: 'Additional arguments passed to the rasa train command'
  required: false
  default: ''
- test_args:
- description: 'Additional arguments passed to the rasa test command'
+ test_nlu_args:
+ description: 'Additional arguments passed to the rasa test nlu command'
+ required: false
+ default: ''
+ test_core_args:
+ description: 'Additional arguments passed to the rasa test core command'
  required: false
  default: ''
  test_type:
@@ -97,16 +109,33 @@ runs:
  mkdir -p ${{ inputs.workspace }}/${{ inputs.result_directory }}
  chmod 777 ${{ inputs.workspace }}/${{ inputs.result_directory }}
 
+ case "${{ inputs.train_type }}" in
+ all)
+ echo TRAIN_TYPE="" >> $GITHUB_ENV
+ ;;
+ nlu)
+ echo TRAIN_TYPE=nlu >> $GITHUB_ENV
+ echo DEFAULT_ARGS="--config /app/${{ inputs.configuration }}" >> $GITHUB_ENV
+ ;;
+ core)
+ echo TRAIN_TYPE=core >> $GITHUB_ENV
+ ;;
+ *)
+ echo "::error::Unknown training type: ${{ inputs.train_type }} (use: core/nlu/all)"
+ exit 1
+ esac
+
  case "${{ inputs.test_type }}" in
  all)
- echo TEST_TYPE="" >> $GITHUB_ENV
+ echo TEST_NLU="true" >> $GITHUB_ENV
+ echo TEST_CORE="true" >> $GITHUB_ENV
  ;;
  nlu)
- echo TEST_TYPE=nlu >> $GITHUB_ENV
+ echo TEST_NLU="true" >> $GITHUB_ENV
  echo DEFAULT_ARGS="--config /app/${{ inputs.configuration }}" >> $GITHUB_ENV
  ;;
  core)
- echo TEST_TYPE=core >> $GITHUB_ENV
+ echo TEST_CORE="true" >> $GITHUB_ENV
  ;;
  *)
  echo "::error::Unknown test type: ${{ inputs.test_type }} (use: core/nlu/all)"
@@ -179,6 +208,16 @@ runs:
  echo "::warning::Data validation is disabled. To turn on data validation set the data_validate parameter to 'true'."
  fi
 
+ - name: Fine Tune
+ shell: bash
+ run: |
+ if [[ "${{ inputs.fine_tune }}" == "true" ]]; then
+ echo "Fine tuning model is enabled."
+ echo FINE_TUNE="--finetune" >> $GITHUB_ENV
+ else
+ echo "::warning::Fine tune is disabled. To turn on fine-tune, set the fine_tune parameter to 'true'."
+ fi
+
  - name: Run the rasa train command
  shell: bash
  run: |
@@ -189,7 +228,7 @@ runs:
  elif [[ "${{ inputs.rasa_train }}" == "true" ]]; then
  echo "Run the rasa train"
  echo ""
- docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} train ${{ env.TEST_TYPE }} ${{ env.DEFAULT_ARGS }} ${{ inputs.train_args }}
+ docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} train ${{ env.TRAIN_TYPE }} ${{ env.DEFAULT_ARGS }} ${{ env.FINE_TUNE }} ${{ inputs.train_args }}
  else
  echo "::warning::rasa train is disabled. To turn on the rasa train set the rasa_train parameter to 'true'."
  fi
@@ -200,7 +239,12 @@ runs:
  if [[ "${{ inputs.rasa_test }}" == "true" ]]; then
  echo "Run the rasa test"
  echo ""
- docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test ${{ env.TEST_TYPE }} ${{ env.MODEL_ARGS }} ${{ env.CROSS_VALIDATION_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_args }}
+ if [[ "${{ env.TEST_NLU }}" == "true" ]]; then
+ docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test nlu ${{ env.MODEL_ARGS }} ${{ env.CROSS_VALIDATION_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_nlu_args }}
+ fi
+ if [[ "${{ env.TEST_CORE }}" == "true" ]]; then
+ docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test core ${{ env.MODEL_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_core_args }}
+ fi
  else
  echo "::warning::rasa test is disabled. To turn on the rasa train set the rasa_test parameter to 'true'."
  fi