store validation result

cli/__init__.py

@@ -9,3 +9,5 @@
 LSRI_CLIENT_ID = "aa0fcc42-096a-4f9d-b871-aceb1a97d174"
 
 __version__ = open(os.path.join(os.path.dirname(os.path.abspath(cli.__file__)), 'VERSION')).read().strip()
+
+SUB_CLI_CONFIG_FILE = ".eva_sub_cli_config.yml"

cli/docker_validator.py

@@ -4,16 +4,21 @@
 import subprocess
 import time
 
-from cli import ETC_DIR
+from cli import ETC_DIR, SUB_CLI_CONFIG_FILE
 from cli.reporter import Reporter
 from ebi_eva_common_pyutils.logger import logging_config
 
+from cli.writable_config import WritableConfig
+
 logger = logging_config.get_logger(__name__)
 
+docker_path = 'docker'
 container_image = 'eva_sub_cli'
 container_validation_dir = '/opt/vcf_validation'
 container_validation_output_dir = '/opt/vcf_validation/vcf_validation_output'
 container_etc_dir = '/opt/cli/etc'
+VALIDATION_RESULTS = 'validation_results'
+READY_FOR_SUBMISSION_TO_EVA = 'ready_for_submission_to_eva'
 
 
 def run_command_with_output(command_description, command, return_process_output=True,
@@ -48,19 +53,33 @@ def run_command_with_output(command_description, command, return_process_output=
 
 class DockerValidator(Reporter):
 
-    def __init__(self, mapping_file, output_dir, metadata_json=None,
-                 metadata_xlsx=None, container_name=container_image, docker_path='docker'):
+    def __init__(self, mapping_file, output_dir, metadata_json=None, metadata_xlsx=None,
+                 container_name=container_image, docker_path='docker', submission_config=None):
         self.docker_path = docker_path
         self.mapping_file = mapping_file
         self.metadata_json = metadata_json
         self.metadata_xlsx = metadata_xlsx
         self.container_name = container_name
         self.spreadsheet2json_conf = os.path.join(ETC_DIR, "spreadsheet2json_conf.yaml")
+        if submission_config:
+            self.sub_config = submission_config
+        else:
+            config_file = os.path.join(output_dir, SUB_CLI_CONFIG_FILE)
+            self.sub_config = WritableConfig(config_file)
         super().__init__(self._find_vcf_file(), output_dir)
 
     def _validate(self):
         self.run_docker_validator()
 
+    def update_config_with_validation_result(self):
+        self.sub_config.set(VALIDATION_RESULTS, value=self.results)
+        self.sub_config.set(READY_FOR_SUBMISSION_TO_EVA, value=self.verify_ready_for_submission_to_eva())
+        self.sub_config.write()
+
+    def verify_ready_for_submission_to_eva(self):
+        # TODO:  check validation results and confirm if they are good enough for submitting to EVA
+        return True
+
     def _find_vcf_file(self):
         vcf_files = []
         with open(self.mapping_file) as open_file:

cli/eva_sub_cli.py

@@ -1,20 +1,69 @@
+import os
 from argparse import ArgumentParser
 
 from ebi_eva_common_pyutils.logger import logging_config
 
+from cli import SUB_CLI_CONFIG_FILE
+from cli.docker_validator import DockerValidator, docker_path, container_image
 from cli.submit import StudySubmitter
+from cli.writable_config import WritableConfig
+
+VALIDATION_OUTPUT_DIR = "validation_output"
+VALIDATE = 'validate'
+SUBMIT = 'submit'
+RESUME = 'resume'
+
+logging_config.add_stdout_handler()
+
+
+def get_docker_validator(vcf_files_mapping, output_dir, metadata_json, metadata_xlsx,
+                         arg_container, arg_docker, sub_config):
+    docker = arg_docker or docker_path
+    container = arg_container or container_image
+    validation_output_dir = os.path.join(output_dir, VALIDATION_OUTPUT_DIR)
+    return DockerValidator(vcf_files_mapping, validation_output_dir, metadata_json, metadata_xlsx,
+                           container, docker, sub_config)
+
 
 if __name__ == "__main__":
-    argparse = ArgumentParser(description='EVA Submission CLI')
-    argparse.add_argument('--submission-dir', required=True, type=str,
-                          help='Full path to the submission directory where all submission info is/will be stored')
-    argparse.add_argument('--resume', action='store_true', default=False, help='resume an existing submission')
-    args = argparse.parse_args()
-
-    logging_config.add_stdout_handler()
-
-    submitter = StudySubmitter()
-    if args.resume:
-        submitter.upload_submission(args.submission_dir)
-    else:
-        submitter.submit(args.submission_dir)
+    argparser = ArgumentParser(description='EVA SUB CLI - to validate and submit a submission')
+    argparser.add_argument('--task', required=True, choices=[VALIDATE, SUBMIT, RESUME],
+                           help='Select a task to perform')
+    argparser.add_argument('--submission_dir', required=True, type=str,
+                           help='Full path to the directory where all processing will be done and submission info is/will be stored')
+    argparser.add_argument("--docker_path", help="Full path to the docker installation, "
+                                                 "not required if docker is available on path", required=False)
+    argparser.add_argument("--container_name", help="Name of the docker container", required=False)
+    argparser.add_argument("--vcf_files_mapping", required=False,
+                           help="csv file with the mappings for vcf files, fasta and assembly report")
+    group = argparser.add_mutually_exclusive_group(required=False)
+    group.add_argument("--metadata_json",
+                       help="Json file that describe the project, analysis, samples and files")
+    group.add_argument("--metadata_xlsx",
+                       help="Excel spreadsheet  that describe the project, analysis, samples and files")
+
+    args = argparser.parse_args()
+
+    # load config
+    config_file_path = os.path.join(args.submission_dir, SUB_CLI_CONFIG_FILE)
+    sub_config = WritableConfig(config_file_path)
+
+    if args.task == RESUME:
+        submitter = StudySubmitter(args.submission_dir, submission_config=sub_config)
+        submitter.upload_submission()
+
+    if args.task == VALIDATE or args.task == SUBMIT:
+        if not args.vcf_files_mapping:
+            raise Exception(f"Please provide csv file with the mappings of vcf files using --vcf_files_mapping")
+        if not args.metadata_json and not args.metadata_xlsx:
+            raise Exception(f"Please provide the file that describes the project, analysis, samples and files "
+                            f"using either --metadata_json or --metadata_xlsx")
+        docker_validator = get_docker_validator(args.vcf_files_mapping, args.submission_dir, args.metadata_json,
+                                                args.metadata_xlsx, args.container_name, args.docker_path, sub_config)
+        docker_validator.validate()
+        docker_validator.create_reports()
+
+        if args.task == SUBMIT:
+            docker_validator.update_config_with_validation_result()
+            submitter = StudySubmitter(args.submission_dir)
+            submitter.submit(args.submission_dir, submission_config=sub_config)

cli/submit.py

@@ -2,46 +2,45 @@
 import os
 
 import requests
-import yaml
 from ebi_eva_common_pyutils.logger import logging_config
 
+from cli import SUB_CLI_CONFIG_FILE
 from cli.auth import get_auth
+from cli.docker_validator import READY_FOR_SUBMISSION_TO_EVA
+from cli.writable_config import WritableConfig
 
 logger = logging_config.get_logger(__name__)
-SUB_CLI_CONFIG_FILE = ".eva-sub-cli-config.yml"
 SUB_CLI_CONFIG_KEY_SUBMISSION_ID = "submission_id"
 SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL = "submission_upload_url"
 SUBMISSION_INITIATE_URL = "http://www.ebi.ac.uk/eva/v1/submission/initiate"
 
 
 class StudySubmitter:
-    def __init__(self, submission_initiate_url=SUBMISSION_INITIATE_URL):
+    def __init__(self, submission_dir, submission_initiate_url=SUBMISSION_INITIATE_URL, submission_config: WritableConfig = None):
         self.auth = get_auth()
         self.submission_initiate_url = submission_initiate_url
-
-    def create_submission_config_file(self, submission_dir, submission_id, submission_upload_url):
-        submission_config_file = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
-        config_data = {
-            SUB_CLI_CONFIG_KEY_SUBMISSION_ID: submission_id,
-            SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL: submission_upload_url
-        }
-        with open(submission_config_file, 'w') as open_file:
-            yaml.safe_dump(config_data, open_file)
-
-    def get_submission_id_and_upload_url(self, submission_dir):
-        submission_config_file = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
-        if submission_config_file:
-            with (open(submission_config_file, 'r') as f):
-                submission_config_data = yaml.safe_load(f)
-                return submission_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_ID], submission_config_data[
-                    SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL]
+        if submission_config:
+            self.sub_config = submission_config
         else:
-            raise FileNotFoundError(f'Could not upload. No config file found for the submission in {submission_dir}.')
+            config_file = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
+            self.sub_config = WritableConfig(config_file)
+
+    def update_config_with_submission_id_and_upload_url(self, submission_id, upload_url):
+        self.sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, value=submission_id)
+        self.sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL, value=upload_url)
+        self.sub_config.write()
 
     # TODO
-    def upload_submission(self, submission_dir, submission_id=None, submission_upload_url=None):
+    def upload_submission(self, submission_id=None, submission_upload_url=None):
+        if not self.sub_config or self.sub_config.is_empty():
+            raise Exception(f'Cannot resume submission. Config file is empty')
+
+        if not self.sub_config[READY_FOR_SUBMISSION_TO_EVA]:
+            raise Exception(f'There are still validation errors that needs to be fixed. Please fix them before uploading')
+
         if not submission_id or not submission_upload_url:
-            submission_id, submission_upload_url = self.get_submission_id_and_upload_url(submission_dir)
+            submission_id = self.sub_config[SUB_CLI_CONFIG_KEY_SUBMISSION_ID]
+            submission_upload_url = self.sub_config[SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL]
         pass
 
     def verify_submission_dir(self, submission_dir):
@@ -50,14 +49,20 @@ def verify_submission_dir(self, submission_dir):
         if not os.access(submission_dir, os.W_OK):
             raise Exception(f"The directory '{submission_dir}' does not have write permissions.")
 
-
     def submit(self, submission_dir):
+        if not self.sub_config[READY_FOR_SUBMISSION_TO_EVA]:
+            raise Exception(f'There are still validation errors that needs to be fixed. Please fix them before submitting.')
+
         self.verify_submission_dir(submission_dir)
         response = requests.post(self.submission_initiate_url,
                                  headers={'Accept': 'application/hal+json',
                                           'Authorization': 'Bearer ' + self.auth.token})
         response.raise_for_status()
         response_json = response.json()
         logger.info("Submission ID {} received!!".format(response_json["submissionId"]))
-        self.create_submission_config_file(submission_dir, response_json["submissionId"], response_json["uploadUrl"])
-        self.upload_submission(submission_dir, response_json["submissionId"], response_json["uploadUrl"])
+
+        # update config with submission id and upload url
+        self.update_config_with_submission_id_and_upload_url(response_json["submissionId"], response_json["uploadUrl"])
+
+        # upload submission
+        self.upload_submission(response_json["submissionId"], response_json["uploadUrl"])

tests/test_submit.py

@@ -8,64 +8,48 @@
 
 from cli import LSRI_CLIENT_ID
 from cli.auth import WebinAuth, LSRIAuth
-from cli.submit import StudySubmitter, SUB_CLI_CONFIG_FILE, SUB_CLI_CONFIG_KEY_SUBMISSION_ID, \
-    SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL
+from cli.docker_validator import READY_FOR_SUBMISSION_TO_EVA
+from cli.eva_sub_cli import SUB_CLI_CONFIG_FILE
+from cli.submit import StudySubmitter, SUB_CLI_CONFIG_KEY_SUBMISSION_ID, SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL
+from cli.writable_config import WritableConfig
 
 
 class TestSubmit(unittest.TestCase):
 
     def setUp(self) -> None:
         self.token = 'a token'
-        with patch('cli.submit.get_auth', return_value=Mock(token=self.token)):
-            self.submitter = StudySubmitter()
         self.test_sub_dir = os.path.join(os.path.dirname(__file__), 'resources', 'test_sub_dir')
         shutil.rmtree(self.test_sub_dir, ignore_errors=True)
+        self.config_file = os.path.join(self.test_sub_dir, SUB_CLI_CONFIG_FILE)
+        with patch('cli.submit.get_auth', return_value=Mock(token=self.token)):
+            self.submitter = StudySubmitter(self.test_sub_dir)
+
 
     def tearDown(self) -> None:
         shutil.rmtree(self.test_sub_dir, ignore_errors=True)
 
-    def test_submit(self):
-        # Mock the response for post-authentication response from eva-submission-ws
-        # see get_auth_response() in LSRIAuth class
-        mock_submit_response = MagicMock()
-        mock_submit_response.status_code = 200
-        mock_submit_response.text = json.dumps({"submissionId": "mock_submission_id",
-                                                'uploadUrl': 'directory to use for upload'})
-
-        # Set the side_effect attribute to return different responses
-        with patch('cli.submit.requests.post', return_value=mock_submit_response) as mock_post:
-            with patch('cli.submit.StudySubmitter.create_submission_config_file'):
-                self.submitter.submit('test_submission_directory')
-        mock_post.assert_called_once_with('http://www.ebi.ac.uk/eva/v1/submission/initiate',
-                                          headers={'Accept': 'application/hal+json', 'Authorization': 'Bearer a token'})
-
-        # TODO: Check that upload_submission was called with submission id
-
     def test_verify_submission_dir(self):
         self.submitter.verify_submission_dir(self.test_sub_dir)
         assert os.path.exists(self.test_sub_dir)
 
-    def test_verify_submission_dircreate_submission_config_file(self):
-        self.submitter.verify_submission_dir(self.test_sub_dir)
-        self.submitter.create_submission_config_file(self.test_sub_dir, 1234, "/sub/upload/url")
-
-        sub_config_file = os.path.join(self.test_sub_dir, SUB_CLI_CONFIG_FILE)
-        assert os.path.exists(sub_config_file)
-
-        with (open(sub_config_file, 'r') as f):
-            sub_config_data = yaml.safe_load(f)
-            assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_ID] == 1234
-            assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL] == "/sub/upload/url"
-
-    def test_get_submission_id_and_upload_url(self):
+    def test_sub_config_file_creation(self):
         self.submitter.verify_submission_dir(self.test_sub_dir)
-        self.submitter.create_submission_config_file(self.test_sub_dir, 1234, "/sub/upload/url")
+        self.submitter.sub_config.set('test_key', value='test_value')
+        self.submitter.sub_config.write()
 
-        submission_id, upload_url = self.submitter.get_submission_id_and_upload_url(self.test_sub_dir)
+        assert os.path.exists(self.config_file)
+        assert self.submitter.sub_config['test_key'] == 'test_value'
 
-        assert submission_id == 1234
-        assert upload_url == "/sub/upload/url"
+    def test_sub_config_passed_as_param(self):
+        with patch('cli.submit.get_auth', return_value=Mock(token=self.token)):
+            sub_config = WritableConfig(self.config_file)
+            submitter = StudySubmitter(self.test_sub_dir, sub_config)
+            submitter.verify_submission_dir(self.test_sub_dir)
+            submitter.sub_config.set('test_key', value='test_value')
+            submitter.sub_config.write()
 
+            assert os.path.exists(self.config_file)
+            assert submitter.sub_config['test_key'] == 'test_value'
 
 
     def test_submit(self):
@@ -75,13 +59,16 @@ def test_submit(self):
             "submissionId": "mock_submission_id",
             "uploadUrl": "directory to use for upload",
         }
+        sub_config = WritableConfig(self.config_file)
+        sub_config.set(READY_FOR_SUBMISSION_TO_EVA, value=True)
+        sub_config.write()
+
         with patch('cli.submit.requests.post', return_value=mock_submit_response) as mock_post:
             self.submitter.submit(self.test_sub_dir)
 
         assert os.path.exists(self.test_sub_dir)
-        sub_config_file = os.path.join(self.test_sub_dir, SUB_CLI_CONFIG_FILE)
-        assert os.path.exists(sub_config_file)
-        with (open(sub_config_file, 'r') as f):
+        assert os.path.exists(self.config_file)
+        with (open(self.config_file, 'r') as f):
             sub_config_data = yaml.safe_load(f)
             assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_ID] == "mock_submission_id"
             assert sub_config_data[SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL] == "directory to use for upload"