From 4ed2f6fc11d16ac01cfbf9e9d0925063688f20fb Mon Sep 17 00:00:00 2001 From: Giulio Starace Date: Fri, 5 Apr 2024 06:06:16 +0200 Subject: [PATCH] Allow for evals with no args (#1517) As raised in #1515, the `args` field of `EvalSpec` is optional. Therefore it is possible for evals with no args to exist. Here `args` is `None`. However, currently our [arg overriding code](https://github.com/openai/evals/blame/main/evals/cli/oaieval.py#L158) mistakingly does not support this API, since it assumes `args` is not `None`. This PR addresses the issue with an if statement. Fixes #1515 --- evals/cli/oaieval.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/evals/cli/oaieval.py b/evals/cli/oaieval.py index 06c3c5f6e5..a8927dda4c 100644 --- a/evals/cli/oaieval.py +++ b/evals/cli/oaieval.py @@ -155,7 +155,11 @@ def to_number(x: str) -> Union[int, float, str]: return {k: to_number(v) for k, v in str_dict.items()} extra_eval_params = parse_extra_eval_params(args.extra_eval_params) - eval_spec.args.update(extra_eval_params) + + if eval_spec.args is None: + eval_spec.args = extra_eval_params + else: + eval_spec.args.update(extra_eval_params) # If the user provided an argument to --completion_args, parse it into a dict here, to be passed to the completion_fn creation **kwargs completion_args = args.completion_args.split(",")