Merge branch 'main' into newptl_fix_validation_in_spellmapper

bene-ges · Dec 9, 2023 · 5e21342 · 5e21342
2 parents 15c554c + c6cf276
commit 5e21342
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/scripts/nlp_language_modeling/convert_hf_llama_to_nemo.py b/scripts/nlp_language_modeling/convert_hf_llama_to_nemo.py
@@ -98,6 +98,9 @@ def load_config(llama_config):
  nemo_config = OmegaConf.load(
  os.path.join(os.path.dirname(__file__), '../../examples/nlp/language_modeling/conf/megatron_llama_config.yaml')
  ).model
+
+ if llama_config.get('rope_theta', None):
+ nemo_config['rotary_base'] = llama_config['rope_theta']
  nemo_config.encoder_seq_length = llama_config['max_position_embeddings']
  nemo_config.num_layers = int(llama_config['num_hidden_layers'])
  nemo_config.hidden_size = llama_config['hidden_size']