fix: Save failed studies status to db (#309)

EBISPOT · Apr 9, 2024 · d5646e8 · d5646e8
1 parent f4505a3
commit d5646e8
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/sumstats_service/app.py b/sumstats_service/app.py
@@ -275,7 +275,9 @@ def process_studies(
     zero_p_values: bool = False,
     bypass: bool = False,
 ):
+    print(f">>> [process_studies] {callback_id=} with {minrows=} {forcevalid=} {bypass=} {zero_p_values=} {file_type=}")
     if endpoints.create_studies(callback_id=callback_id, file_type=file_type, content=content):
+        print(f'endpoints.create_studies: True for {callback_id=}')
         validate_files_in_background.apply_async(
             kwargs={
                 "callback_id": callback_id,
@@ -301,8 +303,7 @@ def validate_files_in_background(
     zero_p_values: bool = False,
     file_type: Union[str, None] = None,
 ):
-    print("[validate_files_in_background]")
-    print(f">>>>>>> {callback_id=} with {minrows=} {forcevalid=} {bypass=} {zero_p_values=} {file_type=}")
+    print(f">>> [validate_files_in_background] {callback_id=} with {minrows=} {forcevalid=} {bypass=} {zero_p_values=} {file_type=}")
 
     print('calling store_validation_method')
     au.store_validation_method(callback_id=callback_id, bypass_validation=forcevalid)
@@ -329,32 +330,39 @@ def validate_files_in_background(
 
 @celery.task(queue=config.CELERY_QUEUE2, options={"queue": config.CELERY_QUEUE2})
 def store_validation_results(results):
+    print(">>> [store_validation_results]")
     if results:
+        print('results: True')
         au.store_validation_results_in_db(results)
 
 
 @celery.task(queue=config.CELERY_QUEUE1, options={"queue": config.CELERY_QUEUE1})
 def remove_payload_files(callback_id):
+    print(">>> [remove_payload_files]")
     au.remove_payload_files(callback_id)
 
 
 @celery.task(queue=config.CELERY_QUEUE1, options={"queue": config.CELERY_QUEUE1})
 def move_files_to_staging(resp):
+    print(">>> [move_files_to_staging]")
     return au.move_files_to_staging(resp)
 
 
 @celery.task(queue=config.CELERY_QUEUE3, options={"queue": config.CELERY_QUEUE3})
 def convert_metadata_to_yaml(gcst_id, is_harmonised_included=True):
+    print(f">>> [convert_metadata_to_yaml] for {gcst_id=}")
     return au.convert_metadata_to_yaml(gcst_id, is_harmonised_included)
 
 
 @celery.task(queue=config.CELERY_QUEUE1, options={"queue": config.CELERY_QUEUE1})
 def delete_globus_endpoint(globus_endpoint_id):
+    print(f">>> [delete_globus_endpoint] for {globus_endpoint_id}")
     return au.delete_globus_endpoint(globus_endpoint_id)
 
 
 @task_failure.connect
 def task_failure_handler(sender=None, **kwargs) -> None:
+    print(">>> [task_failure_handler]")
     subject = f"Celery error in {sender.name}"
     message = """{einfo} Task was called with args: 
                  {args} kwargs: {kwargs}.\n

diff --git a/sumstats_service/resources/api_utils.py b/sumstats_service/resources/api_utils.py
@@ -182,14 +182,13 @@ def validate_files(
 
         if result.returncode != 0:
             print("Error in submitting job: ", result.stderr)
-            return
+        else:
+            print("No error. Command output: ", result.stdout)
 
-        print("Command output: ", result.stdout)
     except Exception as e:
         print('=== EXCEPTION ===')
         print(e)
 
-
     json_out_files = [f for f in glob.glob(os.path.join(wd, "*.json")) if not f.endswith('payload.json')]
     results = {"callbackID": callback_id, "validationList": []}
     if len(json_out_files) > 0:
@@ -200,9 +199,10 @@ def validate_files(
     else:
         results = results_if_failure(callback_id, content)
 
-    print("results: " + json.dumps(results))
+    results_json_dumped = json.dumps(results)
+    print("results: " + results_json_dumped)
     # remove_payload_files(callback_id)
-    return json.dumps(results)
+    return results_json_dumped
 
 
 def write_data_to_path(data, path):