From 7105af6e44ad0cb1e1bca76435b3741f1fff3dad Mon Sep 17 00:00:00 2001 From: Ivica Kostric Date: Wed, 24 Apr 2024 13:28:23 +0200 Subject: [PATCH] Add documentation for NLU training (#257) --- README.md | 2 +- docs/source/architecture.rst | 36 ++++++++++++++++++- .../annotation/joint_bert/joint_bert_train.py | 9 ++--- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 50466a9..fafe918 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Below is the overview of IAI MovieBot 2.0 architecture. Blue components are inhe - Telegram - Flask REST - Flask socket.io - * Natural Language Understanding (NLU) + * Natural Language Understanding (NLU) [[doc](https://iai-moviebot.readthedocs.io/en/latest/architecture.html#natural-language-understanding)] - Rule-based - JointBERT * Dialogue management diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index 7212033..e05f96a 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -11,7 +11,41 @@ The system architecture is shown in the figure below, illustrating the core proc Natural Language Understanding ------------------------------ -The :py:class:`NLU ` component converts the natural language :py:class:`UserUtterance ` into a :py:class:`DialogueAct `. This process, comprising of *intent detection* and *slot filling*, is performed based on the current dialogue state. +The :py:class:`NLU ` component converts the natural language :py:class:`UserUtterance ` into a :py:class:`DialogueAct `. This process, comprising of *intent detection* and *slot filling*, is performed based on the current dialogue state. The component offers two distinct solutions as modules: Rule-Based and Neural using JointBERT. + +Rule-Based NLU +^^^^^^^^^^^^^^ + +The rule-based NLU module utilizes a combination of keyword extraction and heuristics to determine the user's intent and extract slot-value pairs from their utterances. This approach relies on predefined rules and patterns to interpret user input. + +Neural NLU with JointBERT +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Neural NLU module employs JointBERT, a neural model trained for predicting both the intent of the user's utterance and the corresponding slot-value pairs. + +Training the JointBERT Model +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To train the JointBERT model, the provided training script (`moviebot/nlu/annotation/joint_bert/joint_bert_train.py`) can be utilized. This script downloads a pre-trained BERT model (`bert-base-uncased`) and fine-tunes it on a dataset annotated with intents and slot-value pairs. Below is an overview of the training process: + +1. **Data Preparation**: Ensure the dataset is properly formatted with annotations for intents and slot-value pairs. The data path should be specified using the `--data_path` argument in the training script. + +Example dataset format: +```yaml +REVEAL: + - text: "[I absolutely adore](modifier) movies focusing on [martial arts](keywords)." + - text: "Films about [space exploration](keywords) [fascinate me](modifier)." + - text: "[I can't stand](modifier) movies that emphasize on [corporate politics](keywords)." + - text: "[Space adventures](keywords) [always intrigue me](modifier)." +``` + +2. **Model Initialization**: The model is initialized with the number of intent labels and slot labels based on the dataset. Additionally, hyperparameters such as learning rate, weight decay, and maximum epochs may be configured. + +3. **Training**: The training script supports logging with [Wandb](https://wandb.ai/site) for easy monitoring of training progress. + +4. **Model Saving**: After training, the trained model weights are saved to the specified output path (`--model_output_path`). Additionally, metadata including intent and slot names is saved in a JSON file for reference. + +6. **Usage**: Once trained, the JointBERT model can be integrated into the conversational system for natural language understanding tasks, by specifying the model path. Dialogue Manager diff --git a/moviebot/nlu/annotation/joint_bert/joint_bert_train.py b/moviebot/nlu/annotation/joint_bert/joint_bert_train.py index fcc8a8f..2cc5ca0 100644 --- a/moviebot/nlu/annotation/joint_bert/joint_bert_train.py +++ b/moviebot/nlu/annotation/joint_bert/joint_bert_train.py @@ -178,12 +178,13 @@ def save_pretrained(self, path: str) -> None: def parse_arguments(): """Parses the command line arguments.""" parser = argparse.ArgumentParser() - parser.add_argument("--max_epochs", type=int, default=5) - parser.add_argument("--learning_rate", type=float, default=5e-5) - parser.add_argument("--weight_decay", type=float, default=0.0) parser.add_argument( "--model_output_path", type=str, default=_MODEL_OUTPUT_PATH ) + parser.add_argument("--data_path", type=str, default=_DATA_PATH) + parser.add_argument("--max_epochs", type=int, default=5) + parser.add_argument("--learning_rate", type=float, default=5e-5) + parser.add_argument("--weight_decay", type=float, default=0.0) args = parser.parse_args() return args @@ -192,7 +193,7 @@ def parse_arguments(): args = parse_arguments() wandb_logger = WandbLogger(project="JointBERT") - dataset = JointBERTDataset(_DATA_PATH) + dataset = JointBERTDataset(args.data_path) dataloader = DataLoader(dataset, batch_size=8, shuffle=True, num_workers=4) model = JointBERTTrain(