2.3.0

.test/config/config.yaml

@@ -1,14 +1,41 @@
 samples: config/samples.csv
 
-# # Optional parameters
-# params:
-#   bowtie2:
-#     # Optional parameters for bowtie2-build
-#     build: ""
-#     # Optional parameters for bowtie2-align
-#     align: ""
-#   sambamba:
-#     # Optional parameters for sambamba view
-#     view: "--format 'bam' --filter 'mapping_quality >= 30 and not (unmapped or mate_is_unmapped)' "
-#     # Optional parameters for sambamba markdup
-#     markdup: "--remove-duplicates"
+# Optional parameters
+params:
+  # Optional parameters for fastp
+  fastp:
+    # Optional adapters to remove
+    adapters: ""
+    # Optional command line arguments for fastp
+    extra: ""
+  # Optional parameters for fastqc
+  fastqc: ""
+  # Optional parameters for bowtie2
+  bowtie2:
+    # Optional parameters for bowtie2-build
+    build: ""
+    # Optional parameters for bowtie2-align
+    align: ""
+  sambamba:
+    # Optional parameters for sambamba view
+    view: "--format 'bam' "
+    # Optional parameters for sambamba markdup
+    markdup: "--overflow-list-size=500000"
+  picard:
+    # Mapping QC optional parameters
+    metrics: ""
+  # Optional parameters for samtools stats
+  samtools: ""
+  # Optional parameters for multiqc
+  multiqc: "--module picard --module fastqc --module fastp --module samtools --module bowtie2 --module sambamba --zip-data-dir --verbose --no-megaqc-upload --no-ansi --force"
+
+# Optional path to a `genome.csv` file
+genomes: config/genomes.csv
+
+# Internal use only, not described in documentation.
+# deactivate import of fair_genome_indexer pipeline.
+# Requires the file `genome.csv` to be filled.
+# load_fair_genome_indexer: true
+# Deactivate the import of fair_fastqc_multiqc pipeline.
+# Requires to redefine fastqc and multiqc rules.
+# load_fair_fastqc_multiqc: true

.test/config/genomes.csv

@@ -0,0 +1,7 @@
+species,build,release
+saccharomyces_cerevisiae,R64-1-1,111
+homo_sapiens,GRCh38,111
+homo_sapiens,GRCh37,75
+mus_musculus,GRCm39,111
+mus_musculus,GRCm38,102
+salmo_salar,Ssal_v3.1,111

.test/config/samples.csv

@@ -1,3 +1,3 @@
 sample_id,upstream_file,downstream_file,species,build,release
-sac_a,data/reads/a.scerevisiae.1.fq,data/reads/a.scerevisiae.2.fq,saccharomyces_cerevisiae,R64-1-1,110
-sac_b,data/reads/b.scerevisiae.1.fq,,saccharomyces_cerevisiae,R64-1-1,110
+sac_a,data/reads/a.scerevisiae.1.fq,data/reads/a.scerevisiae.2.fq,saccharomyces_cerevisiae,R64-1-1,111
+sac_b,data/reads/b.scerevisiae.1.fq,,saccharomyces_cerevisiae,R64-1-1,111

.test/makefile

@@ -14,8 +14,8 @@ snakefiles := \
 	"../workflow/rules/picard_metrics.smk" \
 	"../workflow/rules/samtools_stats.smk" \
 	"../workflow/rules/fair_genome_indexer_pipeline.smk" \
+	"../workflow/rules/fair_fastqc_multiqc_pipeline.smk" \
 	"../workflow/rules/common.smk" \
-	"../workflow/rules/fastqc.smk" \
 	"../workflow/Snakefile"
 
 
@@ -41,7 +41,7 @@ pipeline.txt: format.txt linter.txt
 	snakemake -s ../workflow/Snakefile \
 		--cores 2 \
 		--software-deployment-method conda \
-		--workflow-profile ../profiles 2>&1 | tee pipeline.txt
+		2>&1 | tee pipeline.txt
 
 
 format.txt:

CHANGELOG.md

@@ -1,3 +1,19 @@
+# 2.3.0
+
+## Features:
+
+* Include fastqc/multiqc as a module
+* Rename meta-wrapper rules to indetify them
+* Define generic genomes
+* Add configuration keys
+* Add MultiQC report label
+
+## Fix:
+
+* Mat&Met typos
+* Multiqc report missing tools
+* test Makefile update for Snakemake 8+
+
 # 2.2.8
 
 ## Features:

config/config.yaml

@@ -1,25 +1,41 @@
 samples: config/samples.csv
 
-# # Optional parameters
-# params:
-#   bowtie2:
-#     # Optional parameters for bowtie2-build
-#     build: ""
-#     # Optional parameters for bowtie2-align
-#     align: ""
-#   sambamba:
-#     # Optional parameters for sambamba view
-#     view: "--format 'bam' --filter 'mapping_quality >= 30 and not (unmapped or mate_is_unmapped)' "
-#     # Optional parameters for sambamba markdup
-#     markdup: "--remove-duplicates --overflow-list-size=500000"
-#   picard:
-#     # Mapping QC optional parameters
-#     metrics: ""
+# Optional parameters
+params:
+  # Optional parameters for fastp
+  fastp:
+    # Optional adapters to remove
+    adapters: ""
+    # Optional command line arguments for fastp
+    extra: ""
+  # Optional parameters for fastqc
+  fastqc: ""
+  # Optional parameters for bowtie2
+  bowtie2:
+    # Optional parameters for bowtie2-build
+    build: ""
+    # Optional parameters for bowtie2-align
+    align: ""
+  sambamba:
+    # Optional parameters for sambamba view
+    view: "--format 'bam' --filter 'mapping_quality >= 30 and not (unmapped or mate_is_unmapped)' "
+    # Optional parameters for sambamba markdup
+    markdup: "--remove-duplicates --overflow-list-size=500000"
+  picard:
+    # Mapping QC optional parameters
+    metrics: ""
+  # Optional parameters for samtools stats
+  samtools: ""
+  # Optional parameters for multiqc
+  multiqc: "--module picard --module fastqc --module fastp --module samtools --module bowtie2 --module sambamba --zip-data-dir --verbose --no-megaqc-upload --no-ansi --force"
 
 # Optional path to a `genome.csv` file
-# genomes: genome.csv
+genomes: config/genomes.csv
 
 # Internal use only, not described in documentation.
 # deactivate import of fair_genome_indexer pipeline.
 # Requires the file `genome.csv` to be filled.
-# load_fair_genome_indexer: true
+# load_fair_genome_indexer: true
+# Deactivate the import of fair_fastqc_multiqc pipeline.
+# Requires to redefine fastqc and multiqc rules.
+# load_fair_fastqc_multiqc: true

config/genomes.csv

@@ -0,0 +1,7 @@
+species,build,release
+saccharomyces_cerevisiae,R64-1-1,111
+homo_sapiens,GRCh38,111
+homo_sapiens,GRCh37,75
+mus_musculus,GRCm39,111
+mus_musculus,GRCm38,102
+salmo_salar,Ssal_v3.1,111

config/samples.csv

@@ -1,2 +1,2 @@
 sample_id,upstream_file,downstream_file,species,build,release
-sac_a,ngs-test-data/reads/a.scerevisiae.1.fq,ngs-test-data/reads/a.scerevisiae.2.fq,saccharomyces_cerevisiae,R64-1-1,110
+sac_a,ngs-test-data/reads/a.scerevisiae.1.fq,ngs-test-data/reads/a.scerevisiae.2.fq,saccharomyces_cerevisiae,R64-1-1,111

workflow/Snakefile

@@ -1,13 +1,17 @@
 include: "rules/common.smk"
 include: "rules/fastp.smk"
-include: "rules/fastqc.smk"
 
 
 if config.get("load_fair_genome_indexer", True):
 
     include: "rules/fair_genome_indexer_pipeline.smk"
 
 
+if config.get("load_fair_fastqc_multiqc", True):
+
+    include: "rules/fair_fastqc_multiqc_pipeline.smk"
+
+
 include: "rules/bowtie2_sambamba_meta.smk"
 include: "rules/picard_metrics.smk"
 include: "rules/samtools_stats.smk"

workflow/report/material_methods.rst

@@ -44,4 +44,4 @@ usage, and resutls can be found on the `Snakemake workflow`_ page.
 :Authors:
     Thibault Dayris
 
-:Version: 2.2.8 of 01/09/2024
+:Version: 2.3.0 of 01/16/2024

workflow/report/multiqc.rst

@@ -1,6 +1,9 @@
-This HTML_ file contains quality reports of both Fastp_ and Picard_.
+This HTML_ file contains quality reports of Fastp_, Picard_, Samtools_, FastQC_, and Bowtie2_.
 It is a stand-alone file, and can be opened in your favorite web-browser.
 
 .. _HTML: https://en.wikipedia.org/wiki/HTML
 .. _Fastp: https://snakemake-wrappers.readthedocs.io/en/v3.3.3/wrappers/fastp.html
+.. _Bowtie2: https://snakemake-wrappers.readthedocs.io/en/v3.3.3/wrappers/bowtie2/align.html
+.. _FastQC: https://snakemake-wrappers.readthedocs.io/en/v3.3.3/wrappers/fastqc.html
+.. _Stats: https://snakemake-wrappers.readthedocs.io/en/v3.3.3/wrappers/samtools/stats.html
 .. _Picard: https://snakemake-wrappers.readthedocs.io/en/v3.3.3/wrappers/picard/collectmultiplemetrics.html

workflow/report/results.rst

@@ -56,26 +56,32 @@ the following files are present:
     │       ├── YYY.bam
     │       └── YYY.bam.bai
     └── QC
-        ├── MultiQC_data.zip
-        ├── MultiQC.html
+        ├── MultiQC_FastQC_data.zip
+        ├── MultiQC_FastQC.html
+        ├── MultiQC_Mapping_data.zip
+        ├── MultiQC_Mapping.html
         ├── report_pe
         │   └── YYY.html
         └── report_se
             └── YYY.html
 
 
-+---------------+---------------------+-----------------------------------------------+
-| Directory     | File Extension      | Content                                       |
-+===============+=====================+===============================================+
-| XXX/Mapping   | `YYY.bam`           | Aligned reads                                 |
++---------------+-----------------------------+-----------------------------------------------+
+| Directory     | File Extension              | Content                                       |
++===============+=============================+===============================================+
+| XXX/Mapping   | `YYY.bam`                   | Aligned reads                                 |
++               +-----------------------------+-----------------------------------------------+
+|               | `YYY.bam.bai`               | Aligned reads index                           |
++---------------+-----------------------------+-----------------------------------------------+
+| QC            | `MultiQC_FastQC_data.zip`   | Zipped figures and tables                     |
++               +----------------------------+-----------------------------------------------+
+|               | `MultiQC_FastQC.html`       | Complete quality report, includes all samples |
++---------------+-----------------------------+-----------------------------------------------+
+| QC            | `MultiQC_Mapping_data.zip`  | Zipped figures and tables                     |
 +               +---------------------+-----------------------------------------------+
-|               | `YYY.bam.bai`       | Aligned reads index                           |
-+---------------+---------------------+-----------------------------------------------+
-| QC            | `MultiQC_data.zip`  | Zipped figures and tables                     |
-+               +---------------------+-----------------------------------------------+
-|               | `MultiQC.html`      | Complete quality report, includes all samples |
-+---------------+---------------------+-----------------------------------------------+
-| QC/report_pe  | `YYY.html`          | Sequence quality report for PE sample `YYY`   |
-+---------------+---------------------+-----------------------------------------------+
-| QC/report_se  | `YYY.html`          | Sequence quality report for SE sample `YYY`   |
-+---------------+---------------------+-----------------------------------------------+
+|               | `MultiQC_Mapping.html`      | Complete quality report, includes all samples |
++---------------+-----------------------------+-----------------------------------------------+
+| QC/report_pe  | `YYY.html`                  | Sequence quality report for PE sample `YYY`   |
++---------------+-----------------------------+-----------------------------------------------+
+| QC/report_se  | `YYY.html`                  | Sequence quality report for SE sample `YYY`   |
++---------------+-----------------------------+-----------------------------------------------+

workflow/rules/bowtie2_sambamba_meta.smk

@@ -1,11 +1,11 @@
-module bowtie2_sambamba:
+module bowtie2_sambamba_metawrapper:
     meta_wrapper:
         "v3.3.3/meta/bio/bowtie2_sambamba"
     config:
         config
 
 
-use rule bowtie2_build from bowtie2_sambamba with:
+use rule bowtie2_build from bowtie2_sambamba_metawrapper with:
     input:
         unpack(get_bowtie2_build_input),
     output:
@@ -33,7 +33,7 @@ use rule bowtie2_build from bowtie2_sambamba with:
         extra=config.get("params", {}).get("bowtie2", {}).get("build", ""),
 
 
-use rule bowtie2_alignment from bowtie2_sambamba with:
+use rule bowtie2_alignment from bowtie2_sambamba_metawrapper with:
     input:
         unpack(get_bowtie2_alignment_input),
     output:
@@ -58,7 +58,7 @@ use rule bowtie2_alignment from bowtie2_sambamba with:
         ),
 
 
-use rule sambamba_sort from bowtie2_sambamba with:
+use rule sambamba_sort from bowtie2_sambamba_metawrapper with:
     input:
         "tmp/bowtie2/{species}.{build}.{release}.{datatype}/{sample}_raw.bam",
     output:
@@ -76,7 +76,7 @@ use rule sambamba_sort from bowtie2_sambamba with:
         "benchmark/sambamba/sort/{species}.{build}.{release}.{datatype}/{sample}.tsv"
 
 
-use rule sambamba_view from bowtie2_sambamba with:
+use rule sambamba_view from bowtie2_sambamba_metawrapper with:
     input:
         "tmp/sambamba/sort/{species}.{build}.{release}.{datatype}/{sample}.bam",
     output:
@@ -101,7 +101,7 @@ use rule sambamba_view from bowtie2_sambamba with:
         ),
 
 
-use rule sambamba_markdup from bowtie2_sambamba with:
+use rule sambamba_markdup from bowtie2_sambamba_metawrapper with:
     input:
         "tmp/sambamba/view/{species}.{build}.{release}.{datatype}/{sample}.bam",
     output:
@@ -123,7 +123,7 @@ use rule sambamba_markdup from bowtie2_sambamba with:
         .get("markdup", "--remove-duplicates --overflow-list-size=500000"),
 
 
-use rule sambamba_index from bowtie2_sambamba with:
+use rule sambamba_index from bowtie2_sambamba_metawrapper with:
     input:
         "results/{species}.{build}.{release}.{datatype}/Mapping/{sample}.bam",
     output:

workflow/rules/common.smk

@@ -374,8 +374,10 @@ def get_fair_bowtie2_mapping_target(
     """
     results: dict[str, list[str]] = {
         "multiqc": [
-            "results/QC/MultiQC.html",
-            "results/QC/MultiQC_data.zip",
+            "results/QC/MultiQC_FastQC.html",
+            "results/QC/MultiQC_FastQC_data.zip",
+            "results/QC/MultiQC_Mapping.html",
+            "results/QC/MultiQC_Mapping_data.zip",
         ],
         "bams": [],
         "bais": [],

workflow/rules/fair_fastqc_multiqc_pipeline.smk

@@ -0,0 +1,15 @@
+module fair_fastqc_multiqc:
+    snakefile:
+        github("tdayris/fair_fastqc_multiqc", path="workflow/Snakefile", tag="1.0.4")
+    config:
+        {
+            "samples": config.get("samples", "config/samples.csv"),
+            "genomes": config.get("genomes", "config/genomes.csv"),
+            "params": {
+                "fastqc": config.get("params", {}).get("fastqc", ""),
+                "multiqc": config.get("params", {}).get("multiqc", {}),
+            },
+        }
+
+
+use rule * from fair_fastqc_multiqc

workflow/rules/fair_genome_indexer_pipeline.smk

@@ -2,7 +2,7 @@ module fair_genome_indexer:
     snakefile:
         github("tdayris/fair_genome_indexer", path="workflow/Snakefile", tag="2.3.2")
     config:
-        {"genomes": config["genomes"]}
+        {"genomes": config.get("genomes", "config/genomes.csv")}
 
 
-use rule * from fair_genome_indexer as fair_genome_indexer_*
+use rule * from fair_genome_indexer