Skip to content

Commit

Permalink
Remove deprecated test
Browse files Browse the repository at this point in the history
  • Loading branch information
lewtun committed Aug 19, 2024
1 parent 338fbb1 commit 2fe3652
Showing 1 changed file with 0 additions and 13 deletions.
13 changes: 0 additions & 13 deletions tests/test_model_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,19 +64,6 @@ def test_default_chat_template(self):
tokenizer = get_tokenizer(self.model_args, DataArguments())
self.assertEqual(tokenizer.chat_template, DEFAULT_CHAT_TEMPLATE)

def test_default_chat_template_no_overwrite(self):
"""
If no chat template is passed explicitly in the config, then for models with a
`default_chat_template` but no `chat_template` we do not set a `chat_template`,
and that we do not change `default_chat_template`
"""
model_args = ModelArguments(model_name_or_path="m-a-p/OpenCodeInterpreter-SC2-7B")
base_tokenizer = AutoTokenizer.from_pretrained("m-a-p/OpenCodeInterpreter-SC2-7B")
processed_tokenizer = get_tokenizer(model_args, DataArguments())

assert getattr(processed_tokenizer, "chat_template") is None
self.assertEqual(base_tokenizer.get_chat_template(), processed_tokenizer.get_chat_template())

def test_chatml_chat_template(self):
chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
tokenizer = get_tokenizer(self.model_args, DataArguments(chat_template=chat_template))
Expand Down

0 comments on commit 2fe3652

Please sign in to comment.