fixed -profile test but still need to put the requirements on docker …

…and the dataset on nf-core/test-datasets
nf-core · Nov 18, 2024 · d18dfc5 · d18dfc5
1 parent a981ca7
commit d18dfc5
Show file tree

Hide file tree

Showing 12 changed files with 38 additions and 28 deletions.
diff --git a/conf/base.config b/conf/base.config
@@ -40,9 +40,9 @@ process {
         time   = { 8.h   * task.attempt }
     }
     withLabel:process_high {
-        cpus   = { check_max( 18    * task.attempt, 'cpus'    ) }
-        memory = { check_max( 72.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 16.h  * task.attempt, 'time'    ) }
+        cpus   = { 12    * task.attempt }
+        memory = { 72.GB * task.attempt }
+        time   = { 16.h  * task.attempt }
     }
     withLabel:process_long {
         time   = { 20.h  * task.attempt }

diff --git a/conf/modules.config b/conf/modules.config
@@ -20,15 +20,15 @@ process {
 
     withName: 'LOAD_RESPONSE' {
         publishDir = [
-            path: { params.save_datasets ? "${params.outdir}/${params.run_id}/datasets" : params.outdir },
+            path: { params.save_datasets ? "${params.path_data}/${params.run_id}/datasets" : params.path_data },
             mode: params.publish_dir_mode,
             saveAs: { filename -> (filename != 'versions.yml' && params.save_datasets) ? filename : null }
         ]
     }
 
     withName: 'CV_SPLIT' {
         publishDir = [
-            path: { params.save_datasets ? "${params.outdir}/${params.run_id}/datasets" : params.outdir },
+            path: { params.save_datasets ? "${params.path_data}/${params.run_id}/datasets" : params.path_data },
             mode: params.publish_dir_mode,
             saveAs: { filename -> (filename != 'versions.yml' && params.save_datasets) ? filename : null }
         ]
@@ -52,7 +52,7 @@ process {
 
     withName: 'HPAM_SPLIT' {
         publishDir = [
-            path: { params.save_datasets ? "${params.outdir}/${params.run_id}/hpams" : params.outdir },
+            path: { params.save_datasets ? "${params.path_data}/${params.run_id}/hpams" : params.path_data },
             mode: params.publish_dir_mode,
             saveAs: { filename -> (filename != 'versions.yml' && params.save_datasets) ? filename : null }
         ]
@@ -61,7 +61,7 @@ process {
 
     withName: 'TRAIN_AND_PREDICT_CV' {
         publishDir = [
-            path: { params.save_datasets ? "${params.outdir}/${params.run_id}/datasets" : params.outdir },
+            path: { params.save_datasets ? "${params.path_data}/${params.run_id}/datasets" : params.path_data },
             mode: params.publish_dir_mode,
             saveAs: { filename -> (filename != 'versions.yml' && params.save_datasets) ? filename : null }
         ]
@@ -107,15 +107,15 @@ process {
 
     withName: 'EVALUATE_FIND_MAX' {
         publishDir = [
-            path: { params.save_datasets ? "${params.outdir}/${params.run_id}/datasets" : params.outdir },
+            path: { params.save_datasets ? "${params.path_data}/${params.run_id}/datasets" : params.path_data },
             mode: params.publish_dir_mode,
             saveAs: { filename -> (filename != 'versions.yml' && params.save_datasets) ? filename : null }
         ]
     }
 
     withName: 'RANDOMIZATION_SPLIT' {
         publishDir = [
-            path: { params.save_datasets ? "${params.outdir}/${params.run_id}/datasets" : params.outdir },
+            path: { params.save_datasets ? "${params.path_data}/${params.run_id}/datasets" : params.path_data },
             mode: params.publish_dir_mode,
             saveAs: { filename -> (filename != 'versions.yml' && params.save_datasets) ? filename : null }
         ]

diff --git a/conf/test.config b/conf/test.config
@@ -13,7 +13,7 @@
 process {
     resourceLimits = [
         cpus: 4,
-        memory: '15.GB',
+        memory: '3.GB',
         time: '1.h'
     ]
 }
@@ -22,12 +22,13 @@ params {
     config_profile_name        = 'Test profile'
     config_profile_description = 'Minimal test dataset to check pipeline function'
 
-    // TODO nf-core: do this for the proper pipeline
     // Input data
     // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets
     // TODO nf-core: Give any required params for the test so that command line flags are not needed
     //input  = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv'
 
-    // Genome references
-    //genome = 'R64-1-1'
+    run_id = 'test_run'
+    dataset_name = 'Toy_Data'
+    n_cv_splits = 2
+
 }
diff --git a/conf/test_full.config b/conf/test_full.config
@@ -22,4 +22,7 @@ params {
 
     // Genome references
     //genome = 'R64-1-1'
+    run_id = 'test_run'
+    dataset_name = 'Toy_Data'
+    n_cv_splits = 5
 }
diff --git a/modules/local/load_response/main.nf b/modules/local/load_response/main.nf
@@ -9,7 +9,7 @@ process LOAD_RESPONSE {
     //    'biocontainers/python:3.8.3' }"
     input:
     val dataset_name
-    val path_data
+    path path_data
     val cross_study_datasets
 
     output:

diff --git a/modules/local/params_check/main.nf b/modules/local/params_check/main.nf
@@ -21,9 +21,10 @@ process PARAMS_CHECK {
     val optim_metric
     val n_cv_splits
     val response_transformation
+    val path_data
 
     output:
-
+    path path_data,     emit: path_data
 
     when:
     task.ext.when == null || task.ext.when

diff --git a/modules/local/predict_full/main.nf b/modules/local/predict_full/main.nf
@@ -11,7 +11,7 @@ process PREDICT_FULL {
     input:
     tuple path(cross_study_datasets), val(model_name), val(test_mode), val(split_id), path(split_dataset), path(hpam_combi)
     val(response_transformation)
-    val(path_data)
+    path(path_data)
 
     output:
     tuple val(test_mode), val(model_name), path('**predictions*.csv'), emit: ch_vis

diff --git a/modules/local/train_and_predict_cv/main.nf b/modules/local/train_and_predict_cv/main.nf
@@ -5,7 +5,7 @@ process TRAIN_AND_PREDICT_CV {
 
     input:
     tuple val(model_name), val(test_mode), path(cv_data), path(hyperparameters)
-    val path_data
+    path path_data
     val response_transformation
 
     output:

diff --git a/nextflow.config b/nextflow.config
@@ -18,19 +18,19 @@
 params {
     // For this pipeline
     run_id = 'my_run'
-    models = 'SimpleNeuralNetwork'
+    models = 'NaiveDrugMeanPredictor'
     baselines = 'NaivePredictor'
     test_mode = 'LPO'
     randomization_mode = 'None'
     randomization_type = 'permutation'
     n_trials_robustness = 0
-    dataset_name = 'GDSC1'
+    dataset_name = 'GDSC2'
     cross_study_datasets = ''
     curve_curator = false
     optim_metric = 'RMSE'
     n_cv_splits = 10
     response_transformation = 'None'
-    path_data = 'data/'
+    path_data = './data'
     save_datasets = false
 
     // Boilerplate options

diff --git a/subworkflows/local/model_testing/main.nf b/subworkflows/local/model_testing/main.nf
@@ -14,6 +14,7 @@ workflow MODEL_TESTING {
     randomizations              // from input
     cross_study_datasets        // from LOAD_RESPONSE
     ch_models                  // from RUN_CV
+    path_data                  // from input
 
     main:
     if (params.cross_study_datasets == '') {
@@ -30,7 +31,7 @@ workflow MODEL_TESTING {
     PREDICT_FULL (
         ch_predict_final,
         params.response_transformation,
-        params.path_data
+        path_data
     )
     ch_vis = PREDICT_FULL.out.ch_vis
 
@@ -59,7 +60,7 @@ workflow MODEL_TESTING {
 
         RANDOMIZATION_TEST (
             ch_randomization,
-            params.path_data,
+            path_data,
             params.randomization_type,
             params.response_transformation
         )
@@ -82,7 +83,7 @@ workflow MODEL_TESTING {
         ch_robustness = ch_best_hpams_per_split_rob.combine(ch_trials_robustness, by: 0)
         ROBUSTNESS_TEST (
             ch_robustness,
-            params.path_data,
+            path_data,
             params.randomization_type,
             params.response_transformation
         )

diff --git a/subworkflows/local/run_cv/main.nf b/subworkflows/local/run_cv/main.nf
@@ -11,9 +11,10 @@ workflow RUN_CV {
     test_modes                      // LPO,LDO,LCO
     models                          // model names for full testing
     baselines                        // model names for comparison
+    path_data                      // path to data
 
     main:
-    LOAD_RESPONSE(params.dataset_name, params.path_data, params.cross_study_datasets)
+    LOAD_RESPONSE(params.dataset_name, path_data, params.cross_study_datasets)
 
     ch_test_modes = channel.from(test_modes)
     ch_data = ch_test_modes.combine(LOAD_RESPONSE.out.response_dataset)
@@ -73,7 +74,7 @@ workflow RUN_CV {
 
     TRAIN_AND_PREDICT_CV (
         ch_test_combis,
-        params.path_data,
+        path_data,
         params.response_transformation
     )
     // [model_name, test_mode, split_id,

diff --git a/workflows/drugresponseeval.nf b/workflows/drugresponseeval.nf
@@ -60,21 +60,24 @@ workflow DRUGRESPONSEEVAL {
         params.curve_curator,
         params.optim_metric,
         params.n_cv_splits,
-        params.response_transformation
+        params.response_transformation,
+        params.path_data
     )
 
     RUN_CV (
         test_modes,
         models,
-        baselines
+        baselines,
+        PARAMS_CHECK.out.path_data
     )
 
     MODEL_TESTING (
         ch_models_baselines,
         RUN_CV.out.best_hpam_per_split,
         randomizations,
         RUN_CV.out.cross_study_datasets,
-        RUN_CV.out.ch_models
+        RUN_CV.out.ch_models,
+        PARAMS_CHECK.out.path_data
     )
 
     VISUALIZATION (