From 21e3de86225ac3a36d3fc63add3e8dadd4f0522a Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Sun, 16 Jul 2023 23:40:58 -0700 Subject: [PATCH 01/11] test command initial commit --- ersilia/cli/commands/test.py | 31 +++++++++++++++++++++++++++---- ersilia/publish/test.py | 27 +++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ersilia/cli/commands/test.py b/ersilia/cli/commands/test.py index 1a17936c2..2bc27c1ef 100644 --- a/ersilia/cli/commands/test.py +++ b/ersilia/cli/commands/test.py @@ -1,8 +1,23 @@ +import os import click +import json +import tempfile from . import ersilia_cli +from ersilia.cli.commands.run import run_cmd +from ersilia.core.base import ErsiliaBase from ...publish.test import ModelTester -from ... import ModelBase + +# need to import the ModelTester class + +# from ersilia.cli import throw_ersilia_exception +from ersilia.utils.exceptions_utils import throw_ersilia_exception + +# from ..utils.exceptions_utils import test_exceptions as texc +from ersilia.utils.exceptions_utils.test_exceptions import WrongCardIdentifierError + +# from ..default import INFORMATION_FILE +from ersilia.default import INFORMATION_FILE def test_cmd(): @@ -15,8 +30,16 @@ def test_cmd(): ) @click.argument("model", type=click.STRING) def test(model): - mdl = ModelBase(model) + mdl = ModelTester(model) model_id = mdl.model_id + + if model_id is None: + echo( + "No model seems to be served. Please run 'ersilia serve ...' before.", + fg="red", + ) + return + mt = ModelTester(model_id=model_id) - click.echo("Checking model information") - mt.run() + # click.echo("Checking model information") + mt.run() # pass in the input here diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 893e10d76..6ccbc3eea 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -1,11 +1,16 @@ import os import json +import click import tempfile +import types +from ..cli import echo from .. import ErsiliaBase from .. import throw_ersilia_exception +from .. import ErsiliaModel from ..utils.exceptions_utils import test_exceptions as texc +from ..core.session import Session from ..default import INFORMATION_FILE @@ -42,9 +47,27 @@ def check_information(self): @throw_ersilia_exception def check_single_input(self): - self.logger.debug("Checking single input") - pass + self.logger.debug("Testing model on the following single smiles input: COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1") + click.echo("Testing model on the following single smiles input: COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1...") + + session = Session(config_json=None) + service_class = session.current_service_class() + + input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" + mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) + result = mdl.run(input=input, output=None, batch_size=100) + + if isinstance(result, types.GeneratorType): + for result in mdl.run(input=input, output=None, batch_size=100): + if result is not None: + echo(json.dumps(result, indent=4)) + else: + echo("Something went wrong", fg="red") + else: + echo(result) + def run(self): self.check_information() self.check_single_input() + print('all tested!') From 6528f8c6c64c24f98ee8cceba74ffcb36facadc1 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Mon, 17 Jul 2023 21:19:53 -0700 Subject: [PATCH 02/11] test command update --- ersilia/cli/commands/test.py | 7 +++++-- ersilia/publish/test.py | 25 ++++++++++++++++--------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ersilia/cli/commands/test.py b/ersilia/cli/commands/test.py index 2bc27c1ef..698281e9e 100644 --- a/ersilia/cli/commands/test.py +++ b/ersilia/cli/commands/test.py @@ -29,7 +29,10 @@ def test_cmd(): help="Test a model and obtain performance metrics", ) @click.argument("model", type=click.STRING) - def test(model): + @click.option( + "-o", "--output", "output", required=False, default=None, type=click.STRING) + + def test(model, output): mdl = ModelTester(model) model_id = mdl.model_id @@ -42,4 +45,4 @@ def test(model): mt = ModelTester(model_id=model_id) # click.echo("Checking model information") - mt.run() # pass in the input here + mt.run(output) # pass in the output here diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 6ccbc3eea..909860229 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -5,6 +5,7 @@ import types from ..cli import echo +from ..io.input import ExampleGenerator from .. import ErsiliaBase from .. import throw_ersilia_exception from .. import ErsiliaModel @@ -46,28 +47,34 @@ def check_information(self): return data @throw_ersilia_exception - def check_single_input(self): - self.logger.debug("Testing model on the following single smiles input: COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1") - click.echo("Testing model on the following single smiles input: COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1...") + def check_single_input(self, output): + # self.logger.debug("Testing model on custom example input with 10 smiles...") + click.echo("Testing model on custom example input with 5 smiles...") session = Session(config_json=None) service_class = session.current_service_class() - input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" + eg = ExampleGenerator(model_id=self.model_id) + input = eg.example(n_samples=5, file_name=None, simple=True) + mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - result = mdl.run(input=input, output=None, batch_size=100) + result = mdl.run(input=input, output=output, batch_size=100) if isinstance(result, types.GeneratorType): - for result in mdl.run(input=input, output=None, batch_size=100): + for result in mdl.run(input=input, output=output, batch_size=100): if result is not None: echo(json.dumps(result, indent=4)) else: echo("Something went wrong", fg="red") + # print('all tested!') else: echo(result) + # need to adjust it to actually put the output into the output file if specified + + print('all tested!') + - def run(self): + def run(self, output): self.check_information() - self.check_single_input() - print('all tested!') + self.check_single_input(output) From f805ef8211611c4ba7331296b6552ae80dee4e11 Mon Sep 17 00:00:00 2001 From: Febie Lin <133082422+febielin@users.noreply.github.com> Date: Tue, 18 Jul 2023 10:04:51 -0700 Subject: [PATCH 03/11] Create dummy --- notebooks/dummy | 1 + 1 file changed, 1 insertion(+) create mode 100644 notebooks/dummy diff --git a/notebooks/dummy b/notebooks/dummy new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/notebooks/dummy @@ -0,0 +1 @@ +hello From 5dd8fc33a508f1ca3a7f4e55253c9dd230efef12 Mon Sep 17 00:00:00 2001 From: pittmanriley <94929717+pittmanriley@users.noreply.github.com> Date: Tue, 18 Jul 2023 10:06:20 -0700 Subject: [PATCH 04/11] Delete dummy --- notebooks/dummy | 1 - 1 file changed, 1 deletion(-) delete mode 100644 notebooks/dummy diff --git a/notebooks/dummy b/notebooks/dummy deleted file mode 100644 index ce0136250..000000000 --- a/notebooks/dummy +++ /dev/null @@ -1 +0,0 @@ -hello From 046e294d15ad1aa63d2f594e2650e21b4c36e033 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Wed, 19 Jul 2023 14:18:57 -0700 Subject: [PATCH 05/11] Test module updates --- ersilia/cli/commands/test.py | 9 +++---- ersilia/publish/test.py | 36 +++++++++++++++++++++---- test/inputs/out.csv | 6 +++++ test/inputs/out.json | 52 ++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 test/inputs/out.csv create mode 100644 test/inputs/out.json diff --git a/ersilia/cli/commands/test.py b/ersilia/cli/commands/test.py index 698281e9e..c08239127 100644 --- a/ersilia/cli/commands/test.py +++ b/ersilia/cli/commands/test.py @@ -3,6 +3,7 @@ import json import tempfile +from ...cli import echo from . import ersilia_cli from ersilia.cli.commands.run import run_cmd from ersilia.core.base import ErsiliaBase @@ -29,18 +30,14 @@ def test_cmd(): help="Test a model and obtain performance metrics", ) @click.argument("model", type=click.STRING) - @click.option( - "-o", "--output", "output", required=False, default=None, type=click.STRING) + @click.option("-o", "--output", "output", required=False, default=None, type=click.STRING) def test(model, output): mdl = ModelTester(model) model_id = mdl.model_id if model_id is None: - echo( - "No model seems to be served. Please run 'ersilia serve ...' before.", - fg="red", - ) + echo("No model seems to be served. Please run 'ersilia serve ...' before.", fg="red") return mt = ModelTester(model_id=model_id) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 909860229..273f5bdee 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -48,8 +48,30 @@ def check_information(self): @throw_ersilia_exception def check_single_input(self, output): + session = Session(config_json=None) + service_class = session.current_service_class() + + click.echo("Testing model on single smiles input...\n") + + input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" + mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) + result = mdl.run(input=input, output=output, batch_size=100) + + if isinstance(result, types.GeneratorType): + for result in mdl.run(input=input, output=output, batch_size=100): + if result is not None: + echo(json.dumps(result, indent=4)) + else: + echo("Something went wrong", fg="red") + else: + echo(result) + + + + @throw_ersilia_exception + def check_example_input(self, output): # self.logger.debug("Testing model on custom example input with 10 smiles...") - click.echo("Testing model on custom example input with 5 smiles...") + click.echo("\nTesting model on input of 5 smiles given by 'ersilia example' output...\n") session = Session(config_json=None) service_class = session.current_service_class() @@ -66,15 +88,19 @@ def check_single_input(self, output): echo(json.dumps(result, indent=4)) else: echo("Something went wrong", fg="red") - # print('all tested!') else: echo(result) - # need to adjust it to actually put the output into the output file if specified - - print('all tested!') + def check_outputs(self): + pass def run(self, output): self.check_information() self.check_single_input(output) + self.check_example_input(output) + +# To do: +# When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this +# test it with normal run and then try the bash run.sh, comparing the two outputs +# run the same file twice and see if it's similar or the same diff --git a/test/inputs/out.csv b/test/inputs/out.csv new file mode 100644 index 000000000..549e32a4e --- /dev/null +++ b/test/inputs/out.csv @@ -0,0 +1,6 @@ +key,input,mw +OBMNJSNZOWALQB-NCQNOWPTSA-N,COc1ccc2nc3CCCCC[C@@H]4C[C@H]4OC(=O)N[C@H](C(=O)N4C[C@@H](C[C@H]4C(=O)N[C@@]4(C[C@H]4C=C)C(=O)NS(=O)(=O)C4CC4)Oc3nc2c1)C(C)(C)C,766.918 +UWWMQHVWAYFCDT-UHFFFAOYSA-N,NCCCCCNCC12CC3CC(CC(C3)C1)C2,250.42999999999995 +YDDXVAXDYKBWDX-UHFFFAOYSA-N,CN=C(NCCSCc1csc(NC(N)=N)n1)NC#N,312.428 +OBMLHUPNRURLOK-XGRAFVIBSA-N,C[C@]12CC[C@H]3[C@@H](CC[C@H]4C[C@@H]5S[C@@H]5C[C@]34C)[C@@H]1CC[C@@H]2O,306.51500000000004 +XVWPFYDMUFBHBF-CLOONOSVSA-N,COC(=O)[C@H](CC(C)C)NC(=O)c1ccc(NC[C@@H](N)CS)cc1-c1cccc2ccccc12,479.6460000000002 diff --git a/test/inputs/out.json b/test/inputs/out.json new file mode 100644 index 000000000..0426441cd --- /dev/null +++ b/test/inputs/out.json @@ -0,0 +1,52 @@ +[ + { + "input": { + "key": "HDOUGSFASVGDCS-UHFFFAOYSA-N", + "input": "NCc1cccnc1", + "text": "NCc1cccnc1" + }, + "output": { + "mw": 108.14399999999998 + } + }, + { + "input": { + "key": "QTRXIFVSTWXRJJ-UHFFFAOYSA-N", + "input": "Cn1c2ncn(CC(=O)Nc3ccc(nn3)-c3ccccc3)c2c(=O)n(C)c1=O", + "text": "Cn1c2ncn(CC(=O)Nc3ccc(nn3)-c3ccccc3)c2c(=O)n(C)c1=O" + }, + "output": { + "mw": 391.3910000000002 + } + }, + { + "input": { + "key": "SGEGOXDYSFKCPT-UHFFFAOYSA-N", + "input": "NC(=O)c1cc2cc(ccc2o1)N1CCN(CCCCc2c[nH]c3ccc(cc23)C#N)CC1", + "text": "NC(=O)c1cc2cc(ccc2o1)N1CCN(CCCCc2c[nH]c3ccc(cc23)C#N)CC1" + }, + "output": { + "mw": 441.5350000000002 + } + }, + { + "input": { + "key": "DOBMPNYZJYQDGZ-UHFFFAOYSA-N", + "input": "Oc1c(Cc2c(O)c3ccccc3oc2=O)c(=O)oc2ccccc12", + "text": "Oc1c(Cc2c(O)c3ccccc3oc2=O)c(=O)oc2ccccc12" + }, + "output": { + "mw": 336.29900000000004 + } + }, + { + "input": { + "key": "MGNVWUDMMXZUDI-UHFFFAOYSA-N", + "input": "OS(=O)(=O)CCCS(O)(=O)=O", + "text": "OS(=O)(=O)CCCS(O)(=O)=O" + }, + "output": { + "mw": 204.225 + } + } +] \ No newline at end of file From b4585646f29e0c36e6ac8fa14ec05911f553dda9 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Wed, 19 Jul 2023 19:21:51 -0700 Subject: [PATCH 06/11] Test module update --- ersilia/publish/test.py | 64 +++++++++++++------ .../utils/exceptions_utils/test_exceptions.py | 13 ++++ test/inputs/out.csv | 6 -- 3 files changed, 56 insertions(+), 27 deletions(-) delete mode 100644 test/inputs/out.csv diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 273f5bdee..c7bd7ad99 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -3,6 +3,7 @@ import click import tempfile import types +import time from ..cli import echo from ..io.input import ExampleGenerator @@ -28,6 +29,8 @@ def __init__(self, model_id, config_json=None): def _read_information(self): json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) self.logger.debug("Reading model information from {0}".format(json_file)) + if not os.path.exists(json_file): + raise texc.InformationFileNotExist(self.model_id) with open(json_file, "r") as f: data = json.load(f) return data @@ -36,6 +39,17 @@ def _prepare_input_files(self): self.logger.debug("Preparing input files for testing purposes...") pass + def _print_output(self, result): + if isinstance(result, types.GeneratorType): + for r in result: + if r is not None: + echo(json.dumps(r, indent=4)) + else: + echo("Something went wrong", fg="red") + else: + echo(result) + + @throw_ersilia_exception def check_information(self): self.logger.debug("Checking that model information is correct") @@ -46,6 +60,7 @@ def check_information(self): raise texc.WrongCardIdentifierError(self.model_id) return data + @throw_ersilia_exception def check_single_input(self, output): session = Session(config_json=None) @@ -56,21 +71,11 @@ def check_single_input(self, output): input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) result = mdl.run(input=input, output=output, batch_size=100) - - if isinstance(result, types.GeneratorType): - for result in mdl.run(input=input, output=output, batch_size=100): - if result is not None: - echo(json.dumps(result, indent=4)) - else: - echo("Something went wrong", fg="red") - else: - echo(result) - + self._print_output(result) @throw_ersilia_exception def check_example_input(self, output): - # self.logger.debug("Testing model on custom example input with 10 smiles...") click.echo("\nTesting model on input of 5 smiles given by 'ersilia example' output...\n") session = Session(config_json=None) @@ -81,26 +86,43 @@ def check_example_input(self, output): mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) result = mdl.run(input=input, output=output, batch_size=100) + self._print_output(result) - if isinstance(result, types.GeneratorType): - for result in mdl.run(input=input, output=output, batch_size=100): - if result is not None: - echo(json.dumps(result, indent=4)) - else: - echo("Something went wrong", fg="red") - else: - echo(result) + @throw_ersilia_exception + def check_consistent_output(self, output): + self.logger.debug("Confirming model produces consistent output...") + click.echo("Confirming model produces consistent output...") - def check_outputs(self): + session = Session(config_json=None) + service_class = session.current_service_class() + + eg = ExampleGenerator(model_id=self.model_id) + input = eg.example(n_samples=5, file_name=None, simple=True) + + mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) + result = mdl.run(input=input, output=output, batch_size=100) + result2 = mdl.run(input=input, output=output, batch_size=100) + + for item1, item2 in zip(result, result2): + if item1 != item2: + print("Model output is inconsistent. Please review output.") + return + print("Model output is consistent!") + + @throw_ersilia_exception + def run_bash(self, output): pass + def run(self, output): self.check_information() self.check_single_input(output) self.check_example_input(output) + self.run_bash(output) + # self.check_consistent_output(output) + # To do: # When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this # test it with normal run and then try the bash run.sh, comparing the two outputs -# run the same file twice and see if it's similar or the same diff --git a/ersilia/utils/exceptions_utils/test_exceptions.py b/ersilia/utils/exceptions_utils/test_exceptions.py index e3892c092..933bef51b 100644 --- a/ersilia/utils/exceptions_utils/test_exceptions.py +++ b/ersilia/utils/exceptions_utils/test_exceptions.py @@ -12,3 +12,16 @@ def __init__(self, model_id): "Check the model information, usually available in a metadata.json file." ) super().__init__(self.message, self.hints) + + +class InformationFileNotExist(ErsiliaError): + def __init__(self, model_id): + self.message = ( + "The eos/dest/{0}/information.json file does not exist.".format( + model_id + ) + ) + self.hints = ( + "Try fetching and serving the model first." + ) + super().__init__(self.message, self.hints) diff --git a/test/inputs/out.csv b/test/inputs/out.csv deleted file mode 100644 index 549e32a4e..000000000 --- a/test/inputs/out.csv +++ /dev/null @@ -1,6 +0,0 @@ -key,input,mw -OBMNJSNZOWALQB-NCQNOWPTSA-N,COc1ccc2nc3CCCCC[C@@H]4C[C@H]4OC(=O)N[C@H](C(=O)N4C[C@@H](C[C@H]4C(=O)N[C@@]4(C[C@H]4C=C)C(=O)NS(=O)(=O)C4CC4)Oc3nc2c1)C(C)(C)C,766.918 -UWWMQHVWAYFCDT-UHFFFAOYSA-N,NCCCCCNCC12CC3CC(CC(C3)C1)C2,250.42999999999995 -YDDXVAXDYKBWDX-UHFFFAOYSA-N,CN=C(NCCSCc1csc(NC(N)=N)n1)NC#N,312.428 -OBMLHUPNRURLOK-XGRAFVIBSA-N,C[C@]12CC[C@H]3[C@@H](CC[C@H]4C[C@@H]5S[C@@H]5C[C@]34C)[C@@H]1CC[C@@H]2O,306.51500000000004 -XVWPFYDMUFBHBF-CLOONOSVSA-N,COC(=O)[C@H](CC(C)C)NC(=O)c1ccc(NC[C@@H](N)CS)cc1-c1cccc2ccccc12,479.6460000000002 From 58411bc163a435d8e0918dbdb9e66538d12a1f60 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Thu, 20 Jul 2023 13:13:52 -0700 Subject: [PATCH 07/11] Test module updates --- ersilia/publish/test.py | 73 ++++++++++++++----- .../utils/exceptions_utils/test_exceptions.py | 8 ++ 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index c7bd7ad99..31ed83a21 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -15,7 +15,7 @@ from ..core.session import Session from ..default import INFORMATION_FILE - +NUM_SAMPLES = 5 class ModelTester(ErsiliaBase): def __init__(self, model_id, config_json=None): @@ -39,6 +39,9 @@ def _prepare_input_files(self): self.logger.debug("Preparing input files for testing purposes...") pass + """ + This helper method was taken from the run.py file, and just prints the output for the user + """ def _print_output(self, result): if isinstance(result, types.GeneratorType): for r in result: @@ -50,6 +53,9 @@ def _print_output(self, result): echo(result) + """ + Check the model information to make sure it's correct. Need to add more here. + """ @throw_ersilia_exception def check_information(self): self.logger.debug("Checking that model information is correct") @@ -60,7 +66,9 @@ def check_information(self): raise texc.WrongCardIdentifierError(self.model_id) return data - + """ + Runs the model on a single smiles string and prints the output, or writes it to specified output file + """ @throw_ersilia_exception def check_single_input(self, output): session = Session(config_json=None) @@ -73,10 +81,13 @@ def check_single_input(self, output): result = mdl.run(input=input, output=output, batch_size=100) self._print_output(result) - + """ + Generates an example input of 5 smiles using the 'example' command, and then tests the model on that input and prints it + to the consol. + """ @throw_ersilia_exception def check_example_input(self, output): - click.echo("\nTesting model on input of 5 smiles given by 'ersilia example' output...\n") + click.echo("\nTesting model on input of 5 smiles given by 'example' command...\n") session = Session(config_json=None) service_class = session.current_service_class() @@ -85,30 +96,56 @@ def check_example_input(self, output): input = eg.example(n_samples=5, file_name=None, simple=True) mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - result = mdl.run(input=input, output=output, batch_size=100) + result = mdl.run(input=input, output=output, batch_size=100) self._print_output(result) + """ + Gets an example input of 5 smiles using the 'example' command, and then runs this same input on the + model twice. Then, it checks if the outputs are consistent or not and specifies that to the user. + Lastly, it makes sure that the number of outputs equals the number of inputs. + """ @throw_ersilia_exception def check_consistent_output(self, output): - self.logger.debug("Confirming model produces consistent output...") - click.echo("Confirming model produces consistent output...") + # self.logger.debug("Confirming model produces consistent output...") + click.echo("\nConfirming model produces consistent output...") session = Session(config_json=None) service_class = session.current_service_class() eg = ExampleGenerator(model_id=self.model_id) - input = eg.example(n_samples=5, file_name=None, simple=True) + input = eg.example(n_samples=NUM_SAMPLES, file_name=None, simple=True) mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) result = mdl.run(input=input, output=output, batch_size=100) result2 = mdl.run(input=input, output=output, batch_size=100) - for item1, item2 in zip(result, result2): - if item1 != item2: - print("Model output is inconsistent. Please review output.") - return - print("Model output is consistent!") + consistent = True + zipped = list(zip(result, result2)) + + for item1, item2 in zipped: + if (item1 != item2): + consistent = False + break + + if consistent: + print("Model output is consistent!") + else: + print("Model output is inconsistent. Please review the output shown below to see if there is an issue with the model. If the differences are small, there may be no issue!\n") + for item1, item2 in zipped: + print(item1) + print(item2) + print('\n') + + click.echo("\nConfirming there are same number of outputs as inputs...") + print("Number of inputs:", NUM_SAMPLES) + print("Number of outputs:", len(zipped)) + + if NUM_SAMPLES != len(zipped): + raise texc.MissingOutputs(self.model_id) + else: + echo("Number of outputs and inputs are equal!") + @throw_ersilia_exception def run_bash(self, output): @@ -119,10 +156,12 @@ def run(self, output): self.check_information() self.check_single_input(output) self.check_example_input(output) - self.run_bash(output) - # self.check_consistent_output(output) + self.check_consistent_output(output) + # self.run_bash(output) # To do: -# When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this -# test it with normal run and then try the bash run.sh, comparing the two outputs +# 1. When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this +# 2. test it with normal run and then try the bash run.sh, comparing the two outputs +# 3. speed it up by making it so the check_consistent_output function doesn't have to re-generate the example input and then run the model on that input twice +# 4. for each major aspect of the test module, I'd like the header when showing the user what it's doing to be more like a header (bold, different color, etc.) diff --git a/ersilia/utils/exceptions_utils/test_exceptions.py b/ersilia/utils/exceptions_utils/test_exceptions.py index 933bef51b..7965c8cfd 100644 --- a/ersilia/utils/exceptions_utils/test_exceptions.py +++ b/ersilia/utils/exceptions_utils/test_exceptions.py @@ -25,3 +25,11 @@ def __init__(self, model_id): "Try fetching and serving the model first." ) super().__init__(self.message, self.hints) + + +class MissingOutputs(ErsiliaError): + def __init__(self, model_id): + self.message = ("There are not as many outputs as there are inputs. They must be the same.\n") + self.hints = ("Check whether or not the code for the model is skipping a header, or if any of the smiles used are not working correctly.") + super().__init__(self.message, self.hints) + From a64ed134f8e145304cf0655c43e3f83190495467 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Tue, 25 Jul 2023 14:36:01 -0700 Subject: [PATCH 08/11] Test module updates --- ersilia/publish/test.py | 253 ++++++++++++++++-- .../utils/exceptions_utils/test_exceptions.py | 44 +-- 2 files changed, 252 insertions(+), 45 deletions(-) diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 31ed83a21..ed415aecd 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -3,7 +3,7 @@ import click import tempfile import types -import time +import subprocess from ..cli import echo from ..io.input import ExampleGenerator @@ -15,7 +15,16 @@ from ..core.session import Session from ..default import INFORMATION_FILE + +RUN_FILE = "run.sh" +DATA_FILE = "data.csv" +OUTPUT_FILE = "output.csv" +FRAMEWORK_BASEDIR = "framework" +DIFFERENCE_THRESHOLD = 5 # outputs should be within this percent threshold to be considered consistent NUM_SAMPLES = 5 +BOLD = '\033[1m' +RESET = '\033[0m' +RED = '\033[31m' class ModelTester(ErsiliaBase): def __init__(self, model_id, config_json=None): @@ -25,6 +34,7 @@ def __init__(self, model_id, config_json=None): self._info = self._read_information() self._input = self._info["card"]["Input"] self._prepare_input_files() + self.RUN_FILE = "run.sh" def _read_information(self): json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) @@ -43,6 +53,7 @@ def _prepare_input_files(self): This helper method was taken from the run.py file, and just prints the output for the user """ def _print_output(self, result): + echo("Printing output...") if isinstance(result, types.GeneratorType): for r in result: if r is not None: @@ -51,20 +62,151 @@ def _print_output(self, result): echo("Something went wrong", fg="red") else: echo(result) + + """ + This helper method checks that the model ID is correct. + """ + def _check_model_id(self, data): + print("Checking model ID...") + if data["card"]["Identifier"] != self.model_id: + raise texc.WrongCardIdentifierError(self.model_id) + + """ + This helper method checks that the slug field is non-empty. + """ + def _check_model_slug(self, data): + print("Checking model slug...") + if not data["card"]["Slug"]: + raise texc.EmptyField("slug") + """ - Check the model information to make sure it's correct. Need to add more here. + This helper method checks that the description field is non-empty. + """ + def _check_model_description(self, data): + print("Checking model description...") + if not data["card"]["Description"]: + raise texc.EmptyField("Description") + + """ + This helper method checks that the model task is one of the following valid entries: + - Classification + - Regression + - Generative + - Representation + - Similarity + - Clustering + - Dimensionality reduction + """ + def _check_model_task(self, data): + print("Checking model task...") + valid_tasks = [[ 'Classification'], [ 'Regression' ], [ 'Generative' ], [ 'Representation' ], + [ 'Similarity' ], [ 'Clustering' ], [ 'Dimensionality reduction' ]] + if data["card"]["Task"] not in valid_tasks: + raise texc.InvalidEntry("Task") + + """ + This helper method checks that the input field is one of the following valid entries: + - Compound + - Protein + - Text + """ + def _check_model_input(self, data): + print("Checking model input...") + valid_inputs = [[ 'Compound' ], [ 'Protein' ], [ 'Text' ]] + if data["card"]["Input"] not in valid_inputs: + raise texc.InvalidEntry("Input") + + """ + This helper method checks that the input shape field is one of the following valid entries: + - Single + - Pair + - List + - Pair of Lists + - List of Lists + """ + def _check_model_input_shape(self, data): + print("Checking model input shape...") + valid_input_shapes = ["Single", "Pair", "List", "Pair of Lists", "List of Lists"] + if data["card"]["Input Shape"] not in valid_input_shapes: + raise texc.InvalidEntry("Input Shape") + + """ + This helper method checks the the output is one of the following valid entries: + - Boolean + - Compound + - Descriptor + - Distance + - Experimental value + - Image + - Other value + - Probability + - Protein + - Score + - Text + """ + def _check_model_output(self, data): + print("Checking model output...") + valid_outputs = [[ 'Boolean' ], [ 'Compound' ], [ 'Descriptor' ], [ 'Distance' ], [ 'Experimental value' ], + [ 'Image' ], [ 'Other value' ], [ 'Probability' ], [ 'Protein' ], [ 'Score' ], [ 'Text' ]] + if data["card"]["Output"] not in valid_outputs: + raise texc.InvalidEntry("Output") + + """ + This helper method checks that the output type is one of the following valid entries: + - String + - Float + - Integer + """ + def _check_model_output_type(self, data): + print("Checking model output type...") + valid_output_types = [[ 'String' ], [ 'Float' ], [ 'Integer' ]] + if data["card"]["Output Type"] not in valid_output_types: + raise texc.InvalidEntry("Output Type") + + """ + This helper method checks that the output shape is one of the following valid entries: + - Single + - List + - Flexible List + - Matrix + - Serializable Object + """ + def _check_model_output_shape(self, data): + print("Checking model output shape...") + valid_output_shapes = ["Single", "List", "Flexible List", "Matrix", "Serializable Object"] + if data["card"]["Output Shape"] not in valid_output_shapes: + raise texc.InvalidEntry("Output Shape") + + """ + Check the model information to make sure it's correct. Performs the following checks: + - Checks that model ID is correct + - Checks that model slug is non-empty + - Checks that model description is non-empty + - Checks that the model task is valid + - Checks that the model input, input shape is valid + - Checks that the model output, output type, output shape is valid """ @throw_ersilia_exception def check_information(self): self.logger.debug("Checking that model information is correct") + print(BOLD + "Beginning checks for {0} model information:".format(self.model_id) + RESET) json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) with open(json_file, "r") as f: data = json.load(f) - if data["card"]["Identifier"] != self.model_id: - raise texc.WrongCardIdentifierError(self.model_id) - return data + + self._check_model_id(data) + self._check_model_slug(data) + self._check_model_description(data) + self._check_model_task(data) + self._check_model_input(data) + self._check_model_input_shape(data) + self._check_model_output(data) + self._check_model_output_type(data) + self._check_model_output_shape(data) + print("SUCCESS! Model information verified.\n") + """ Runs the model on a single smiles string and prints the output, or writes it to specified output file @@ -74,7 +216,7 @@ def check_single_input(self, output): session = Session(config_json=None) service_class = session.current_service_class() - click.echo("Testing model on single smiles input...\n") + click.echo(BOLD + "Testing model on single smiles input...\n" + RESET) input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) @@ -87,7 +229,7 @@ def check_single_input(self, output): """ @throw_ersilia_exception def check_example_input(self, output): - click.echo("\nTesting model on input of 5 smiles given by 'example' command...\n") + click.echo(BOLD + "\nTesting model on input of 5 smiles given by 'example' command...\n" + RESET) session = Session(config_json=None) service_class = session.current_service_class() @@ -108,7 +250,7 @@ def check_example_input(self, output): @throw_ersilia_exception def check_consistent_output(self, output): # self.logger.debug("Confirming model produces consistent output...") - click.echo("\nConfirming model produces consistent output...") + click.echo(BOLD + "\nConfirming model produces consistent output..." + RESET) session = Session(config_json=None) service_class = session.current_service_class() @@ -123,45 +265,102 @@ def check_consistent_output(self, output): consistent = True zipped = list(zip(result, result2)) - for item1, item2 in zipped: - if (item1 != item2): - consistent = False - break - if consistent: - print("Model output is consistent!") - else: - print("Model output is inconsistent. Please review the output shown below to see if there is an issue with the model. If the differences are small, there may be no issue!\n") - for item1, item2 in zipped: - print(item1) - print(item2) - print('\n') + """ + *** + We can only check for this 5% threshold if the outputs are numbers. If strings, it won't work + *** + """ + - click.echo("\nConfirming there are same number of outputs as inputs...") + for item1, item2 in zipped: + output1 = item1['output']['mw'] + output2 = item2['output']['mw'] + # check to see if the first and second outputs are within 5% from each other + if (100 * (abs(output2 - output1) / ((output1 + output2) / 2)) > DIFFERENCE_THRESHOLD): + for item1, item2 in zipped: + print(item1) + print(item2) + print('\n') + raise texc.InconsistentOutputs(self.model_id) + + print("Model output is consistent!") + + click.echo(BOLD + "\nConfirming there are same number of outputs as inputs..." + RESET) print("Number of inputs:", NUM_SAMPLES) print("Number of outputs:", len(zipped)) if NUM_SAMPLES != len(zipped): - raise texc.MissingOutputs(self.model_id) + raise texc.MissingOutputs() else: echo("Number of outputs and inputs are equal!") - + @throw_ersilia_exception def run_bash(self, output): - pass + print("Running the model bash script...") + # Save current directory - atm, this must be run from root directory (~) + current_dir = os.getcwd() + + # Create temp directory and clone model + with tempfile.TemporaryDirectory() as temp_dir: + repo_url = 'https://github.com/ersilia-os/{0}.git'.format(self.model_id) # Replace with the actual GitHub repository URL + try: + subprocess.run(['git', 'clone', repo_url, temp_dir], check=True) + except subprocess.CalledProcessError as e: + print(f"Error while cloning the repository: {e}") + + # Navigate into the temporary directory + subdirectory_path = os.path.join(temp_dir, "model/framework") + os.chdir(subdirectory_path) + + # Create temp file + with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: + temp_file_path = temp_file.name + + # Run bash script with specified args + output_path = temp_file_path + run_path = os.path.join(temp_dir, "model/framework/run.sh") # path to run.sh + arg1 = os.path.join(current_dir, "ersilia/test/inputs/compound_singles.csv") # input + arg2 = output_path # output + + try: + subprocess.run(['bash', run_path, ".", arg1, arg2,], check=True) + except subprocess.CalledProcessError as e: + print(f"Error while running the bash script: {e}") + + with open(output_path, 'r') as temp_file: + output_contents = temp_file.read() + + print("Output contents:") + print(output_contents) + + # maybe instead of printing the contents of the bash, we can run just compare it with an ersilia run + + + @throw_ersilia_exception + def run_using_bash(self): + tmp_folder = tempfile.mkdtemp(prefix="eos-") + run_file = os.path.join(tmp_folder, self.RUN_FILE) + data_file = os.path.join(tmp_folder, DATA_FILE) + output_file = os.path.join(tmp_folder, OUTPUT_FILE) + + cur_path = os.path.dirname(os.path.realpath(__file__)) + framework_dir = os.path.join(cur_path, "..", "..", "..", self.model_id, "model", FRAMEWORK_BASEDIR, "run.sh") + subprocess.run(["chmod +x " + framework_dir, framework_dir, data_file, output_file], shell=True) def run(self, output): self.check_information() self.check_single_input(output) self.check_example_input(output) self.check_consistent_output(output) + # self.run_using_bash() # self.run_bash(output) # To do: -# 1. When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this +# 1. When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this + # to do this, write each output to a temporary file, and then append all of them to a final file and delete the temporary files at the end # 2. test it with normal run and then try the bash run.sh, comparing the two outputs -# 3. speed it up by making it so the check_consistent_output function doesn't have to re-generate the example input and then run the model on that input twice -# 4. for each major aspect of the test module, I'd like the header when showing the user what it's doing to be more like a header (bold, different color, etc.) +# 3. Make sure the output matches the expectation of the otuput (ex: if it expects a float, it actually gets a float) diff --git a/ersilia/utils/exceptions_utils/test_exceptions.py b/ersilia/utils/exceptions_utils/test_exceptions.py index 7965c8cfd..fab7f96ad 100644 --- a/ersilia/utils/exceptions_utils/test_exceptions.py +++ b/ersilia/utils/exceptions_utils/test_exceptions.py @@ -3,33 +3,41 @@ class WrongCardIdentifierError(ErsiliaError): def __init__(self, model_id): - self.message = ( - "The model identifier in the model card is not correct: {0}".format( - model_id - ) - ) - self.hints = ( - "Check the model information, usually available in a metadata.json file." - ) + self.message = ("The model identifier in the model card is not correct: {0}".format(model_id)) + self.hints = ("Check the model information, usually available in a metadata.json file.") super().__init__(self.message, self.hints) class InformationFileNotExist(ErsiliaError): def __init__(self, model_id): - self.message = ( - "The eos/dest/{0}/information.json file does not exist.".format( - model_id - ) - ) - self.hints = ( - "Try fetching and serving the model first." - ) + self.message = ("The eos/dest/{0}/information.json file does not exist.".format(model_id)) + self.hints = ("Try fetching and serving the model first, and make sure the model is written correctly.") super().__init__(self.message, self.hints) class MissingOutputs(ErsiliaError): - def __init__(self, model_id): + def __init__(self): self.message = ("There are not as many outputs as there are inputs. They must be the same.\n") - self.hints = ("Check whether or not the code for the model is skipping a header, or if any of the smiles used are not working correctly.") + self.hints = ("Try checking whether or not the code for the model is skipping a header, or if any of the smiles used are not working correctly.") + super().__init__(self.message, self.hints) + + +class InconsistentOutputs(ErsiliaError): + def __init__(self, model_id): + self.message = ("Model outputs are inconsistent, meaning the outputs do not fit the 5% similarity threshold.") + self.hints = ("Observe the output comparisons for each smiles input above.") + super().__init__(self.message, self.hints) + + +class EmptyField(ErsiliaError): + def __init__(self, empty_field): + self.message = ("The {0} field in the model card is empty.".format(empty_field)) + self.hints = ("Check the model information, usually available in a metadata.json file.") + super().__init__(self.message, self.hints) + +class InvalidEntry(ErsiliaError): + def __init__(self, invalid_field): + self.message = ("The {0} field of this model is not recognized as a valid format.".format(invalid_field)) + self.hints = ("Check the model information, usually available in a metadata.json file.") super().__init__(self.message, self.hints) From 79a00bb6bc3ec5132577b17480d1d96be9c6a6b6 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Tue, 8 Aug 2023 09:43:02 -0700 Subject: [PATCH 09/11] Test module updates --- ersilia/publish/test.py | 330 +++++++++++++----- .../utils/exceptions_utils/test_exceptions.py | 6 + test/inputs/out.json | 52 --- 3 files changed, 247 insertions(+), 141 deletions(-) delete mode 100644 test/inputs/out.json diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index ed415aecd..8dfd30f11 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -4,8 +4,12 @@ import tempfile import types import subprocess +import time +import re from ..cli import echo +from fuzzywuzzy import fuzz +from fuzzywuzzy import process from ..io.input import ExampleGenerator from .. import ErsiliaBase from .. import throw_ersilia_exception @@ -18,23 +22,27 @@ RUN_FILE = "run.sh" DATA_FILE = "data.csv" -OUTPUT_FILE = "output.csv" -FRAMEWORK_BASEDIR = "framework" DIFFERENCE_THRESHOLD = 5 # outputs should be within this percent threshold to be considered consistent NUM_SAMPLES = 5 BOLD = '\033[1m' RESET = '\033[0m' -RED = '\033[31m' + class ModelTester(ErsiliaBase): def __init__(self, model_id, config_json=None): ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) self.model_id = model_id + self.model_size = 0 self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") self._info = self._read_information() self._input = self._info["card"]["Input"] self._prepare_input_files() self.RUN_FILE = "run.sh" + self.information_check = False + self.single_input = False + self.example_input = False + self.consistent_output = False + self.run_using_bash = False def _read_information(self): json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) @@ -49,20 +57,54 @@ def _prepare_input_files(self): self.logger.debug("Preparing input files for testing purposes...") pass + """ + This function uses the scikit vectorizer package to compare the differences between outputs when + they're strings and not floats + """ + def _compare_output_strings(self, output1, output2): + ratio = fuzz.ratio(output1, output2) + return ratio + + def _is_below_difference_threshold(self, output1, output2): + if output1 == 0.0 or output2 == 0.0: + return output1 == output2 + elif output1 is None or output2 is None: + return output1 == output2 + else: + return (100 * (abs(output1 - output2) / ((output1 + output2) / 2)) < DIFFERENCE_THRESHOLD) + + + def _set_model_size(self, directory): + for dirpath, dirnames, filenames in os.walk(directory): + for filename in filenames: + file_path = os.path.join(dirpath, filename) + self.model_size += os.path.getsize(file_path) + """ This helper method was taken from the run.py file, and just prints the output for the user """ - def _print_output(self, result): + def _print_output(self, result, output): echo("Printing output...") - if isinstance(result, types.GeneratorType): - for r in result: - if r is not None: - echo(json.dumps(r, indent=4)) - else: - echo("Something went wrong", fg="red") + + if isinstance(result, types.GeneratorType): + for r in result: + if r is not None: + if output is not None: + with open(output.name, 'w') as file: + json.dump(r, output.name) + else: + echo(json.dumps(r, indent=4)) + else: + if output is not None: + message = echo("Something went wrong", fg="red") + with open(output.name, 'w') as file: + json.dump(message, output.name) + else: + echo("Something went wrong", fg="red") + else: echo(result) - + """ This helper method checks that the model ID is correct. @@ -179,6 +221,7 @@ def _check_model_output_shape(self, data): if data["card"]["Output Shape"] not in valid_output_shapes: raise texc.InvalidEntry("Output Shape") + """ Check the model information to make sure it's correct. Performs the following checks: - Checks that model ID is correct @@ -189,7 +232,7 @@ def _check_model_output_shape(self, data): - Checks that the model output, output type, output shape is valid """ @throw_ersilia_exception - def check_information(self): + def check_information(self, output): self.logger.debug("Checking that model information is correct") print(BOLD + "Beginning checks for {0} model information:".format(self.model_id) + RESET) json_file = os.path.join(self._dest_dir, self.model_id, INFORMATION_FILE) @@ -207,6 +250,9 @@ def check_information(self): self._check_model_output_shape(data) print("SUCCESS! Model information verified.\n") + if output is not None: + self.information_check = True + """ Runs the model on a single smiles string and prints the output, or writes it to specified output file @@ -215,13 +261,16 @@ def check_information(self): def check_single_input(self, output): session = Session(config_json=None) service_class = session.current_service_class() + input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" click.echo(BOLD + "Testing model on single smiles input...\n" + RESET) - - input = "COc1ccc2c(NC(=O)Nc3cccc(C(F)(F)F)n3)ccnc2c1" mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) result = mdl.run(input=input, output=output, batch_size=100) - self._print_output(result) + + if output is not None: + self.single_input = True + else: + self._print_output(result, output) """ Generates an example input of 5 smiles using the 'example' command, and then tests the model on that input and prints it @@ -229,17 +278,19 @@ def check_single_input(self, output): """ @throw_ersilia_exception def check_example_input(self, output): - click.echo(BOLD + "\nTesting model on input of 5 smiles given by 'example' command...\n" + RESET) - session = Session(config_json=None) service_class = session.current_service_class() - eg = ExampleGenerator(model_id=self.model_id) - input = eg.example(n_samples=5, file_name=None, simple=True) + input = eg.example(n_samples=NUM_SAMPLES, file_name=None, simple=True) + click.echo(BOLD + "\nTesting model on input of 5 smiles given by 'example' command...\n" + RESET) mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) result = mdl.run(input=input, output=output, batch_size=100) - self._print_output(result) + + if output is not None: + self.example_input = True + else: + self._print_output(result, output) """ @@ -248,7 +299,7 @@ def check_example_input(self, output): Lastly, it makes sure that the number of outputs equals the number of inputs. """ @throw_ersilia_exception - def check_consistent_output(self, output): + def check_consistent_output(self): # self.logger.debug("Confirming model produces consistent output...") click.echo(BOLD + "\nConfirming model produces consistent output..." + RESET) @@ -258,32 +309,63 @@ def check_consistent_output(self, output): eg = ExampleGenerator(model_id=self.model_id) input = eg.example(n_samples=NUM_SAMPLES, file_name=None, simple=True) - mdl = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) - result = mdl.run(input=input, output=output, batch_size=100) - result2 = mdl.run(input=input, output=output, batch_size=100) + mdl1 = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) + mdl2 = ErsiliaModel(self.model_id, service_class=service_class, config_json=None) + result = mdl1.run(input=input, output=None, batch_size=100) + result2 = mdl2.run(input=input, output=None, batch_size=100) - consistent = True zipped = list(zip(result, result2)) - - """ - *** - We can only check for this 5% threshold if the outputs are numbers. If strings, it won't work - *** - """ - - for item1, item2 in zipped: - output1 = item1['output']['mw'] - output2 = item2['output']['mw'] - # check to see if the first and second outputs are within 5% from each other - if (100 * (abs(output2 - output1) / ((output1 + output2) / 2)) > DIFFERENCE_THRESHOLD): - for item1, item2 in zipped: - print(item1) - print(item2) - print('\n') - raise texc.InconsistentOutputs(self.model_id) - + output1 = item1["output"] + output2 = item2["output"] + + keys1 = list(output1.keys()) + keys2 = list(output2.keys()) + + for key1, key2 in zip(keys1, keys2): + + # check if the output types are not the same + if not isinstance(output1[key1], type(output2[key2])): + for item1, item2 in zipped: + print(item1) + print(item2) + print('\n') + raise texc.InconsistentOutputTypes(self.model_id) + + if output1[key1] is None: + continue + + elif isinstance(output1[key1], (float, int)): + + # check to see if the first and second outputs are within 5% from each other + if not self._is_below_difference_threshold(output1[key1], output2[key2]): + for item1, item2 in zipped: + print(item1) + print(item2) + print('\n') + raise texc.InconsistentOutputs(self.model_id) + elif isinstance(output1[key1], list): + ls1 = output1[key1] + ls2 = output2[key2] + + for elem1, elem2 in zip(ls1, ls2): + if isinstance(elem1, float): # if one of the outputs is a float, then that means the other is a float too + if not self._is_below_difference_threshold(elem1, elem2): + for item1, item2 in zipped: + print(item1) + print(item2) + print('\n') + raise texc.InconsistentOutputs(self.model_id) + else: + if self._compare_output_strings(elem1, elem2) <= 0.95: + raise texc.InconsistentOutputs(self.model_id) + else: + # if it reaches this, then the outputs are just strings + if self._compare_output_strings(output1[key1], output2[key2]) <= 0.95: + raise texc.InconsistentOutputs(self.model_id) + + self.consistent_output = True print("Model output is consistent!") click.echo(BOLD + "\nConfirming there are same number of outputs as inputs..." + RESET) @@ -296,71 +378,141 @@ def check_consistent_output(self, output): echo("Number of outputs and inputs are equal!") + def _parse_dockerfile(self, temp_dir, pyversion): + packages = set() + prefix = "FROM bentoml/model-server:0.11.0-py" + os.chdir(temp_dir) # navigate into cloned repo + with open('Dockerfile', 'r') as dockerfile: + lines = dockerfile.readlines() + assert lines[0].startswith(prefix) + pyversion[0] = lines[0][len(prefix):] + lines_as_string = '\n'.join(lines) + run_lines = re.findall(r'^\s*RUN\s+(.+)$', lines_as_string, re.MULTILINE) + return run_lines + + # WITH CONDA!!!! @throw_ersilia_exception - def run_bash(self, output): + def run_bash(self): print("Running the model bash script...") - # Save current directory - atm, this must be run from root directory (~) + + # Save current working directory - atm, this must be run from root directory (~) + # TODO: is there a way to change this so that this test command doesn't have to be run from root dir current_dir = os.getcwd() # Create temp directory and clone model with tempfile.TemporaryDirectory() as temp_dir: - repo_url = 'https://github.com/ersilia-os/{0}.git'.format(self.model_id) # Replace with the actual GitHub repository URL + repo_url = 'https://github.com/ersilia-os/{0}.git'.format(self.model_id) try: subprocess.run(['git', 'clone', repo_url, temp_dir], check=True) except subprocess.CalledProcessError as e: print(f"Error while cloning the repository: {e}") + # halt this check if the run.sh file does not exist (e.g. eos3b5e) + if not os.path.exists(os.path.join(temp_dir, "model/framework/run.sh")): + print("Check halted: run.sh file does not exist.") + return + # Navigate into the temporary directory subdirectory_path = os.path.join(temp_dir, "model/framework") os.chdir(subdirectory_path) - # Create temp file - with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: - temp_file_path = temp_file.name - - # Run bash script with specified args - output_path = temp_file_path - run_path = os.path.join(temp_dir, "model/framework/run.sh") # path to run.sh - arg1 = os.path.join(current_dir, "ersilia/test/inputs/compound_singles.csv") # input - arg2 = output_path # output + # Parse Dockerfile + #dockerfile_path = os.path.join(temp_dir, "Dockerfile") + pyversion = [0] + packages = self._parse_dockerfile(temp_dir, pyversion) + pyversion[0] = pyversion[0][0] + '.' + pyversion[0][1:] + conda_env_name = self.model_id try: - subprocess.run(['bash', run_path, ".", arg1, arg2,], check=True) - except subprocess.CalledProcessError as e: - print(f"Error while running the bash script: {e}") + # subprocess.run(['conda', 'create', '-n', self.model_id, 'python={0}'.format(pyversion[0])], check=True) + subprocess.run(['conda', 'create', '-n', self.model_id, 'python={0}'.format('3.10.0')], check=True) + subprocess.run(['conda', 'activate', conda_env_name], shell=True, check=True) + + # install packages + for package in packages: + if 'conda install' in package: + # Handle conda package installation + subprocess.run(package, shell=True, check=True) + elif 'pip install' in package: + subprocess.run(package, shell=True, check=True) + else: + print("Invalid package command:", package) + print("Packages printed!") + + # Create temp file + with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: + temp_file_path = temp_file.name + + # Run bash script with specified args + output_path = temp_file_path + run_path = os.path.join(temp_dir, "model/framework/run.sh") # path to run.sh + arg1 = os.path.join(current_dir, "ersilia/test/inputs/compound_singles.csv") # input + arg2 = output_path # output + + try: + subprocess.run(['bash', run_path, ".", arg1, arg2,], check=True) + except subprocess.CalledProcessError as e: + print(f"Error while running the bash script: {e}") + + with open(output_path, 'r') as temp_file: + output_contents = temp_file.read() + + print("Output contents:") + print(output_contents) + + deactivate_command = "conda deactivate" + subprocess.run(deactivate_command, shell=True, check=True) + + except Exception as e: + print(f"Error while creating or activating the conda environment: {e}") - with open(output_path, 'r') as temp_file: - output_contents = temp_file.read() - - print("Output contents:") - print(output_contents) - - # maybe instead of printing the contents of the bash, we can run just compare it with an ersilia run - - - @throw_ersilia_exception - def run_using_bash(self): - tmp_folder = tempfile.mkdtemp(prefix="eos-") - run_file = os.path.join(tmp_folder, self.RUN_FILE) - data_file = os.path.join(tmp_folder, DATA_FILE) - output_file = os.path.join(tmp_folder, OUTPUT_FILE) - - cur_path = os.path.dirname(os.path.realpath(__file__)) - framework_dir = os.path.join(cur_path, "..", "..", "..", self.model_id, "model", FRAMEWORK_BASEDIR, "run.sh") - - subprocess.run(["chmod +x " + framework_dir, framework_dir, data_file, output_file], shell=True) + + """ + writes to the .json file all the basic information received from the test module: + - size of the model + - did the basic checks pass? True or False + - time to run the model + - did the single input run without error? True or False + - did the run bash run without error? True or False + - did the example input run without error? True or False + - are the outputs consistent? True or False + """ + def make_output(self, output, time): + size_kb = self.model_size / 1024 + size_mb = size_kb / 1024 + size_gb = size_mb / 1024 + + data = {"model size": {"KB": size_kb, "MB": size_mb, "GB": size_gb}, + "time to run model (seconds)": time, + "basic checks passed": self.information_check, + "single input run without error": self.single_input, + "example input run without error": self.example_input, + "outputs consistent": self.consistent_output, + "bash run without error": self.run_using_bash + } + with open(output, "w") as json_file: + json.dump(data, json_file, indent=4) + + + def run(self, output_file): + start = time.time() + self.check_information(output_file) + self.check_single_input(output_file) + self.check_example_input(output_file) + self.check_consistent_output() + # self.run_bash() + + end = time.time() + seconds_taken = end - start + + if output_file is not None: + self.make_output(output_file, seconds_taken) - def run(self, output): - self.check_information() - self.check_single_input(output) - self.check_example_input(output) - self.check_consistent_output(output) - # self.run_using_bash() - # self.run_bash(output) # To do: -# 1. When it currently prints to an output file, it writes the single output result, then deletes that, then prints the result for the example input. Fix this - # to do this, write each output to a temporary file, and then append all of them to a final file and delete the temporary files at the end -# 2. test it with normal run and then try the bash run.sh, comparing the two outputs -# 3. Make sure the output matches the expectation of the otuput (ex: if it expects a float, it actually gets a float) +# 1. test it with normal run and then try the bash run.sh, comparing the two outputs +# 2. Record CPU and memory usage (computational resources) + +# scikit vectorizer: use this to measure distance between two strings +# https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html diff --git a/ersilia/utils/exceptions_utils/test_exceptions.py b/ersilia/utils/exceptions_utils/test_exceptions.py index fab7f96ad..e0c32f85a 100644 --- a/ersilia/utils/exceptions_utils/test_exceptions.py +++ b/ersilia/utils/exceptions_utils/test_exceptions.py @@ -41,3 +41,9 @@ def __init__(self, invalid_field): self.hints = ("Check the model information, usually available in a metadata.json file.") super().__init__(self.message, self.hints) +class InconsistentOutputTypes(ErsiliaError): + def __init__(self, model_id): + self.message = ("Model output types are inconsistent.") + self.hints = ("Observe the output comparisons above for each input, and pay attention to the type of the output (string, float, list, etc.) because they do not match.") + super().__init__(self.message, self.hints) + diff --git a/test/inputs/out.json b/test/inputs/out.json deleted file mode 100644 index 0426441cd..000000000 --- a/test/inputs/out.json +++ /dev/null @@ -1,52 +0,0 @@ -[ - { - "input": { - "key": "HDOUGSFASVGDCS-UHFFFAOYSA-N", - "input": "NCc1cccnc1", - "text": "NCc1cccnc1" - }, - "output": { - "mw": 108.14399999999998 - } - }, - { - "input": { - "key": "QTRXIFVSTWXRJJ-UHFFFAOYSA-N", - "input": "Cn1c2ncn(CC(=O)Nc3ccc(nn3)-c3ccccc3)c2c(=O)n(C)c1=O", - "text": "Cn1c2ncn(CC(=O)Nc3ccc(nn3)-c3ccccc3)c2c(=O)n(C)c1=O" - }, - "output": { - "mw": 391.3910000000002 - } - }, - { - "input": { - "key": "SGEGOXDYSFKCPT-UHFFFAOYSA-N", - "input": "NC(=O)c1cc2cc(ccc2o1)N1CCN(CCCCc2c[nH]c3ccc(cc23)C#N)CC1", - "text": "NC(=O)c1cc2cc(ccc2o1)N1CCN(CCCCc2c[nH]c3ccc(cc23)C#N)CC1" - }, - "output": { - "mw": 441.5350000000002 - } - }, - { - "input": { - "key": "DOBMPNYZJYQDGZ-UHFFFAOYSA-N", - "input": "Oc1c(Cc2c(O)c3ccccc3oc2=O)c(=O)oc2ccccc12", - "text": "Oc1c(Cc2c(O)c3ccccc3oc2=O)c(=O)oc2ccccc12" - }, - "output": { - "mw": 336.29900000000004 - } - }, - { - "input": { - "key": "MGNVWUDMMXZUDI-UHFFFAOYSA-N", - "input": "OS(=O)(=O)CCCS(O)(=O)=O", - "text": "OS(=O)(=O)CCCS(O)(=O)=O" - }, - "output": { - "mw": 204.225 - } - } -] \ No newline at end of file From 1d1d247b04d3d9faa2144d63b2eaf66da1ca97c7 Mon Sep 17 00:00:00 2001 From: Riley Pittman Date: Tue, 8 Aug 2023 21:49:39 -0700 Subject: [PATCH 10/11] Test module updates --- ersilia/cli/commands/test.py | 5 -- ersilia/publish/test.py | 133 +++++++++++++++++++++-------------- 2 files changed, 82 insertions(+), 56 deletions(-) diff --git a/ersilia/cli/commands/test.py b/ersilia/cli/commands/test.py index c08239127..e60261826 100644 --- a/ersilia/cli/commands/test.py +++ b/ersilia/cli/commands/test.py @@ -9,15 +9,10 @@ from ersilia.core.base import ErsiliaBase from ...publish.test import ModelTester -# need to import the ModelTester class - -# from ersilia.cli import throw_ersilia_exception from ersilia.utils.exceptions_utils import throw_ersilia_exception -# from ..utils.exceptions_utils import test_exceptions as texc from ersilia.utils.exceptions_utils.test_exceptions import WrongCardIdentifierError -# from ..default import INFORMATION_FILE from ersilia.default import INFORMATION_FILE diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 8dfd30f11..abfe780d3 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -9,15 +9,12 @@ from ..cli import echo from fuzzywuzzy import fuzz -from fuzzywuzzy import process from ..io.input import ExampleGenerator from .. import ErsiliaBase from .. import throw_ersilia_exception from .. import ErsiliaModel - from ..utils.exceptions_utils import test_exceptions as texc from ..core.session import Session - from ..default import INFORMATION_FILE RUN_FILE = "run.sh" @@ -36,7 +33,6 @@ def __init__(self, model_id, config_json=None): self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") self._info = self._read_information() self._input = self._info["card"]["Input"] - self._prepare_input_files() self.RUN_FILE = "run.sh" self.information_check = False self.single_input = False @@ -52,19 +48,23 @@ def _read_information(self): with open(json_file, "r") as f: data = json.load(f) return data - - def _prepare_input_files(self): - self.logger.debug("Preparing input files for testing purposes...") - pass - + """ - This function uses the scikit vectorizer package to compare the differences between outputs when - they're strings and not floats + This function uses the fuzzy wuzzy package to compare the differences between outputs when + they're strings and not floats. The fuzz.ratio gives the percent of similarity between the two outputs. + Example: two strings that are the exact same will return 100 """ def _compare_output_strings(self, output1, output2): - ratio = fuzz.ratio(output1, output2) - return ratio + if output1 is None and output2 is None: + return 100 + else: + return fuzz.ratio(output1, output2) + """ + To compare outputs, we are stating that numbers generated by the models need to be within 5% of each + other in order to be considered consistent. This function returns true if the outputs are within that + 5% threshold (meaning they're consistent), and false if they are not (meaning they are not consistent). + """ def _is_below_difference_threshold(self, output1, output2): if output1 == 0.0 or output2 == 0.0: return output1 == output2 @@ -74,6 +74,10 @@ def _is_below_difference_threshold(self, output1, output2): return (100 * (abs(output1 - output2) / ((output1 + output2) / 2)) < DIFFERENCE_THRESHOLD) + """ + When the user specifies an output file, the file will show the user how big the model is. This function + calculates the size of the model to allow this. + """ def _set_model_size(self, directory): for dirpath, dirnames, filenames in os.walk(directory): for filename in filenames: @@ -143,10 +147,17 @@ def _check_model_description(self, data): """ def _check_model_task(self, data): print("Checking model task...") - valid_tasks = [[ 'Classification'], [ 'Regression' ], [ 'Generative' ], [ 'Representation' ], - [ 'Similarity' ], [ 'Clustering' ], [ 'Dimensionality reduction' ]] - if data["card"]["Task"] not in valid_tasks: - raise texc.InvalidEntry("Task") + valid_tasks = [ 'Classification', 'Regression', 'Generative' , 'Representation', + 'Similarity', 'Clustering', 'Dimensionality reduction'] + sep = ', ' + tasks = [] + if sep in data["card"]["Task"]: + tasks = data["card"]["Task"].split(sep) + else: + tasks = data["card"]["Task"] + for task in tasks: + if task not in valid_tasks: + raise texc.InvalidEntry("Task") """ This helper method checks that the input field is one of the following valid entries: @@ -190,10 +201,17 @@ def _check_model_input_shape(self, data): """ def _check_model_output(self, data): print("Checking model output...") - valid_outputs = [[ 'Boolean' ], [ 'Compound' ], [ 'Descriptor' ], [ 'Distance' ], [ 'Experimental value' ], - [ 'Image' ], [ 'Other value' ], [ 'Probability' ], [ 'Protein' ], [ 'Score' ], [ 'Text' ]] - if data["card"]["Output"] not in valid_outputs: - raise texc.InvalidEntry("Output") + valid_outputs = [ 'Boolean', 'Compound', 'Descriptor', 'Distance', 'Experimental value', + 'Image', 'Other value', 'Probability', 'Protein', 'Score', 'Text'] + sep = ', ' + outputs = [] + if sep in data["card"]["Output"]: + outputs = data["card"]["Output"].split(sep) + else: + outputs = data["card"]["Output"] + for output in outputs: + if output not in valid_outputs: + raise texc.InvalidEntry("Output") """ This helper method checks that the output type is one of the following valid entries: @@ -221,6 +239,22 @@ def _check_model_output_shape(self, data): if data["card"]["Output Shape"] not in valid_output_shapes: raise texc.InvalidEntry("Output Shape") + """ + This is a helper function for the run_bash() function, and it parses through the Dockerfile to find + the package installation lines. + """ + def _parse_dockerfile(self, temp_dir, pyversion): + packages = set() + prefix = "FROM bentoml/model-server:0.11.0-py" + os.chdir(temp_dir) # navigate into cloned repo + with open('Dockerfile', 'r') as dockerfile: + lines = dockerfile.readlines() + assert lines[0].startswith(prefix) + pyversion[0] = lines[0][len(prefix):] + lines_as_string = '\n'.join(lines) + run_lines = re.findall(r'^\s*RUN\s+(.+)$', lines_as_string, re.MULTILINE) + return run_lines + """ Check the model information to make sure it's correct. Performs the following checks: @@ -255,7 +289,7 @@ def check_information(self, output): """ - Runs the model on a single smiles string and prints the output, or writes it to specified output file + Runs the model on a single smiles string and prints to the user if no output is specified. """ @throw_ersilia_exception def check_single_input(self, output): @@ -274,7 +308,7 @@ def check_single_input(self, output): """ Generates an example input of 5 smiles using the 'example' command, and then tests the model on that input and prints it - to the consol. + to the consol if no output file is specified by the user. """ @throw_ersilia_exception def check_example_input(self, output): @@ -295,8 +329,9 @@ def check_example_input(self, output): """ Gets an example input of 5 smiles using the 'example' command, and then runs this same input on the - model twice. Then, it checks if the outputs are consistent or not and specifies that to the user. - Lastly, it makes sure that the number of outputs equals the number of inputs. + model twice. Then, it checks if the outputs are consistent or not and specifies that to the user. If + it is not consistent, an InconsistentOutput error is raised. Lastly, it makes sure that the number of + outputs equals the number of inputs. """ @throw_ersilia_exception def check_consistent_output(self): @@ -319,7 +354,7 @@ def check_consistent_output(self): for item1, item2 in zipped: output1 = item1["output"] output2 = item2["output"] - + keys1 = list(output1.keys()) keys2 = list(output2.keys()) @@ -344,6 +379,7 @@ def check_consistent_output(self): print(item1) print(item2) print('\n') + # maybe change it to print all of the outputs, and in the error raised it highlights exactly the ones that were off raise texc.InconsistentOutputs(self.model_id) elif isinstance(output1[key1], list): ls1 = output1[key1] @@ -359,10 +395,14 @@ def check_consistent_output(self): raise texc.InconsistentOutputs(self.model_id) else: if self._compare_output_strings(elem1, elem2) <= 0.95: + print('output1 value:', elem1) + print('output2 value:', elem2) raise texc.InconsistentOutputs(self.model_id) else: # if it reaches this, then the outputs are just strings if self._compare_output_strings(output1[key1], output2[key2]) <= 0.95: + print('output1 value:', output1[key1]) + print('output2 value:', output2[key2]) raise texc.InconsistentOutputs(self.model_id) self.consistent_output = True @@ -375,25 +415,14 @@ def check_consistent_output(self): if NUM_SAMPLES != len(zipped): raise texc.MissingOutputs() else: - echo("Number of outputs and inputs are equal!") + echo("Number of outputs and inputs are equal!\n") - - def _parse_dockerfile(self, temp_dir, pyversion): - packages = set() - prefix = "FROM bentoml/model-server:0.11.0-py" - os.chdir(temp_dir) # navigate into cloned repo - with open('Dockerfile', 'r') as dockerfile: - lines = dockerfile.readlines() - assert lines[0].startswith(prefix) - pyversion[0] = lines[0][len(prefix):] - lines_as_string = '\n'.join(lines) - run_lines = re.findall(r'^\s*RUN\s+(.+)$', lines_as_string, re.MULTILINE) - return run_lines # WITH CONDA!!!! @throw_ersilia_exception def run_bash(self): - print("Running the model bash script...") + # print("Running the model bash script...") + print("Cloning a temporary file and calculating model size...") # Save current working directory - atm, this must be run from root directory (~) # TODO: is there a way to change this so that this test command doesn't have to be run from root dir @@ -407,6 +436,17 @@ def run_bash(self): except subprocess.CalledProcessError as e: print(f"Error while cloning the repository: {e}") + # we will remove this part later, but will keep until we get the run_bash() function working + self._set_model_size(temp_dir) + size_kb = self.model_size / 1024 + size_mb = size_kb / 1024 + size_gb = size_mb / 1024 + print("\nModel Size:") + print("KB:", size_kb) + print("MB:", size_mb) + print("GB:", size_gb) + return + # halt this check if the run.sh file does not exist (e.g. eos3b5e) if not os.path.exists(os.path.join(temp_dir, "model/framework/run.sh")): print("Check halted: run.sh file does not exist.") @@ -483,7 +523,7 @@ def make_output(self, output, time): size_gb = size_mb / 1024 data = {"model size": {"KB": size_kb, "MB": size_mb, "GB": size_gb}, - "time to run model (seconds)": time, + "time to run tests (seconds)": time, "basic checks passed": self.information_check, "single input run without error": self.single_input, "example input run without error": self.example_input, @@ -500,19 +540,10 @@ def run(self, output_file): self.check_single_input(output_file) self.check_example_input(output_file) self.check_consistent_output() - # self.run_bash() + self.run_bash() end = time.time() seconds_taken = end - start if output_file is not None: self.make_output(output_file, seconds_taken) - - - -# To do: -# 1. test it with normal run and then try the bash run.sh, comparing the two outputs -# 2. Record CPU and memory usage (computational resources) - -# scikit vectorizer: use this to measure distance between two strings -# https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html From 2f139bac9add43b1b8e23d71ae415cef61fdb2a2 Mon Sep 17 00:00:00 2001 From: gemmaturon Date: Wed, 9 Aug 2023 11:11:23 +0200 Subject: [PATCH 11/11] Add fuzzywuzzy req --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 41a240765..5c69046e6 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ def get_version(package_path): "tqdm", "click", "docker", + "fuzzywuzzy" ] slim_requires = slim