Merge pull request #1595 from nf-core/dev

Release 3.4.3

.editorconfig

@@ -28,10 +28,6 @@ indent_style = unset
 [/assets/email*]
 indent_size = unset
 
-# ignore Readme
-[README.md]
-indent_style = unset
-
-# ignore python
+# ignore python and markdown
 [*.{py,md}]
 indent_style = unset

.github/PULL_REQUEST_TEMPLATE.md

@@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/sare
 - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/sarek/tree/master/.github/CONTRIBUTING.md)
 - [ ] If necessary, also make a PR on the nf-core/sarek _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
-- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --profile +docker`).
+- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
 - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.
 - [ ] Output Documentation in `docs/output.md` is updated.

.github/workflows/ci.yml

@@ -70,7 +70,7 @@ jobs:
 
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v4
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
 
       - name: Hash Github Workspace
         id: hash_workspace
@@ -89,7 +89,7 @@ jobs:
         run: pip install --upgrade -r tests/requirements.txt
 
       - name: Install Nextflow ${{ matrix.NXF_VER }}
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
@@ -214,6 +214,7 @@ jobs:
           - "23.04.0"
           - "latest-everything"
         exclude:
+          - tags: "bcftools/annotate"
           - tags: "bcftools/concat"
           - tags: "bcftools/mpileup"
           - tags: "bcftools/sort"
@@ -223,7 +224,9 @@ jobs:
           - tags: "bwamem2/mem"
           - tags: "cat/cat"
           - tags: "cat/fastq"
+          - tags: "cnvkit/antitarget"
           - tags: "cnvkit/batch"
+          - tags: "cnvkit/reference"
           - tags: "deepvariant"
           - tags: "dragmap/align"
           - tags: "dragmap/hashtable"
@@ -243,9 +246,12 @@ jobs:
           - tags: "gatk4/mutect2"
           - tags: "gatk4spark/applybqsr"
           - tags: "gatk4spark/markduplicates"
+          - tags: "gawk"
           - tags: "mosdepth"
           - tags: "multiqc"
+          - tags: "ngscheckmate/ncm"
           - tags: "samblaster"
+          - tags: "samtools/convert"
           - tags: "samtools/mpileup"
           - tags: "samtools/stats"
           - tags: "snpeff/snpeff"
@@ -274,7 +280,7 @@ jobs:
           java-version: "17"
 
       - name: Install Nextflow ${{ matrix.NXF_VER }}
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 

.github/workflows/download_pipeline.yml

@@ -14,6 +14,8 @@ on:
   pull_request:
     types:
       - opened
+      - edited
+      - synchronize
     branches:
       - master
   pull_request_target:
@@ -28,11 +30,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
 
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+      - name: Disk space cleanup
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           architecture: "x64"
       - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7
         with:
@@ -65,7 +70,16 @@ jobs:
       - name: Inspect download
         run: tree ./${{ env.REPOTITLE_LOWERCASE }}
 
-      - name: Run the downloaded pipeline
+      - name: Run the downloaded pipeline (stub)
+        id: stub_run_pipeline
+        continue-on-error: true
+        env:
+          NXF_SINGULARITY_CACHEDIR: ./
+          NXF_SINGULARITY_HOME_MOUNT: true
+        run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results
+      - name: Run the downloaded pipeline (stub run not supported)
+        id: run_pipeline
+        if: ${{ job.steps.stub_run_pipeline.status == failure() }}
         env:
           NXF_SINGULARITY_CACHEDIR: ./
           NXF_SINGULARITY_HOME_MOUNT: true

.github/workflows/fix-linting.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Use the @nf-core-bot token to check out so we can push later
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
         with:
           token: ${{ secrets.nf_core_bot_auth_token }}
 
@@ -32,9 +32,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
 
       # Install and run pre-commit
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          python-version: 3.11
+          python-version: "3.12"
 
       - name: Install pre-commit
         run: pip install pre-commit

.github/workflows/linting.yml

@@ -14,13 +14,12 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
-      - name: Set up Python 3.11
-        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+      - name: Set up Python 3.12
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          python-version: 3.11
-          cache: "pip"
+          python-version: "3.12"
 
       - name: Install pre-commit
         run: pip install pre-commit
@@ -32,14 +31,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
 
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           architecture: "x64"
 
       - name: Install dependencies
@@ -60,7 +59,7 @@ jobs:
 
       - name: Upload linting log file artifact
         if: ${{ always() }}
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4
         with:
           name: linting-logs
           path: |

.github/workflows/linting_comment.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download lint results
-        uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3
+        uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3
         with:
           workflow: linting.yml
           workflow_conclusion: completed

.github/workflows/release-announcements.yml

@@ -12,7 +12,7 @@ jobs:
       - name: get topics and convert to hashtags
         id: get_topics
         run: |
-          curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT
+          echo "topics=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ')"  >> $GITHUB_OUTPUT
 
       - uses: rzr/fediverse-action@master
         with:
@@ -25,13 +25,13 @@ jobs:
 
             Please see the changelog: ${{ github.event.release.html_url }}
 
-            ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics
+            ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics
 
   send-tweet:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
           python-version: "3.10"
       - name: Install dependencies

.gitignore

@@ -10,3 +10,5 @@ testing*
 .nf-test*
 .nf-test/
 test-datasets/
+test.tap
+test.xml

.nf-core.yml

@@ -1,19 +1,16 @@
 repository_type: pipeline
+nf_core_version: "2.14.1"
 lint:
   actions_ci: False
   files_exist:
     - .github/workflows/awsfulltest.yml
     - .github/workflows/awstest.yml
     - conf/modules.config
-    - lib/WorkflowMain.groovy
-    - lib/NfcoreTemplate.groovy
-    - lib/WorkflowSarek.groovy
   files_unchanged:
     - .gitignore
     - .github/PULL_REQUEST_TEMPLATE.md
     - assets/nf-core-sarek_logo_light.png
     - docs/images/nf-core-sarek_logo_dark.png
     - docs/images/nf-core-sarek_logo_light.png
-    - lib/NfcoreTemplate.groovy
-    - lib/NfcoreSchema.groovy
+  modules_config: False
   template_strings: False

.pre-commit-config.yaml

@@ -3,6 +3,9 @@ repos:
     rev: "v3.1.0"
     hooks:
       - id: prettier
+        additional_dependencies:
+          - prettier@3.2.5
+
   - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
     rev: "2.7.3"
     hooks:

CHANGELOG.md

@@ -5,6 +5,57 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.4.3](https://github.com/nf-core/sarek/releases/tag/3.4.3) - Loametjåhkkå
+
+Loametjåhkkå is another one of the main peaks of the Pårte massif.
+
+### Added
+
+- [#1502](https://github.com/nf-core/sarek/pull/1502) - export CNVs into VCF format in `bam_variant_calling_cnvkit`
+- [#1534](https://github.com/nf-core/sarek/pull/1534), [#1573](https://github.com/nf-core/sarek/pull/1573) - Handling `.fastq.gz.spring` files as input
+- [#1593](https://github.com/nf-core/sarek/pull/1593) - Prepare release `3.4.2`
+
+### Changed
+
+- [#1502](https://github.com/nf-core/sarek/pull/1502) - Improved handling of CNVkit reference
+- [#1502](https://github.com/nf-core/sarek/pull/1502) - Specific CNV call step, with recommended settings for germline
+- [#1508](https://github.com/nf-core/sarek/pull/1508) - Sync `TEMPLATE` with `tools` `2.14.0`
+- [#1513](https://github.com/nf-core/sarek/pull/1513) - Back to dev
+- [#1518](https://github.com/nf-core/sarek/pull/1518) - Sync `TEMPLATE` with `tools` `2.14.1`
+- [#1521](https://github.com/nf-core/sarek/pull/1521) - Minor code refactoring to simplify syntax in args handling
+- [#1545](https://github.com/nf-core/sarek/pull/1545) - Update modules
+- [#1552](https://github.com/nf-core/sarek/pull/1552) - Update samtools to v1.20
+- [#1545](https://github.com/nf-core/sarek/pull/1545) - Update modules
+- [#1553](https://github.com/nf-core/sarek/pull/1553) - Update bcftools to v1.20
+- [#1557](https://github.com/nf-core/sarek/pull/1557) - Update ENSEMBLVEP cache to 111
+
+### Fixed
+
+- [#1536](https://github.com/nf-core/sarek/pull/1536) - Correct typo `Strelka2` to `Strelka`
+- [#1541](https://github.com/nf-core/sarek/pull/1541) - Getting bam and bai published in the same folder
+- [#1542](https://github.com/nf-core/sarek/pull/1542) - Removing legacy configs of `CUSTOM_DUMPSOFTWAREVERSIONS`
+- [#1547](https://github.com/nf-core/sarek/pull/1547) - Correct typo in help text in nextflow_schema.json
+- [#1556](https://github.com/nf-core/sarek/pull/1556) - Fix display of some commands in `docs/usage.md`
+- [#1563](https://github.com/nf-core/sarek/pull/1563) - Fix `vep_cache_path_full` so that `--refseq/--merged` will work for ENSEMBLVEP
+- [#1570](https://github.com/nf-core/sarek/pull/1570) - Remove duplicated notes in FASTQC output docs
+- [#1596](https://github.com/nf-core/sarek/pull/1596) - Fix haplotypecaller tests
+- [#1597](https://github.com/nf-core/sarek/pull/1597) - Fix deepvariant tests
+- [#1612](https://github.com/nf-core/sarek/pull/1612) - Remove empty output directories
+
+### Removed
+
+### Dependencies
+
+| Dependency | Old version | New version |
+| ---------- | ----------- | ----------- |
+| `bcftools` | 1.18        | 1.20        |
+| `bwa`      | 0.7.17      | 0.7.18      |
+| `cnvkit`   | 0.9.10      | 0.9.11      |
+| `htslib`   | 1.19.1      | 1.20        |
+| `samtools` | 1.19.2      | 1.20        |
+
+### Parameters
+
 ## [3.4.2](https://github.com/nf-core/sarek/releases/tag/3.4.2) - Sájtáristjåhkkå
 
 Sájtáristjåhkkå is another peak (just under 2k) in the Pårte massif, it is one of the few peaks in Sweden that cannot be climbed without proper climbing equipment.

README.md

@@ -16,7 +16,7 @@
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
-[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/sarek)
+[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/nf-core/sarek)
 
 [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23sarek-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/sarek)
 [![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)
@@ -142,12 +142,14 @@ We thank the following people for their extensive assistance in the development
 - [Chela James](https://github.com/chelauk)
 - [David Mas-Ponte](https://github.com/davidmasp)
 - [Edmund Miller](https://github.com/edmundmiller)
+- [Famke Bäuerle](https://github.com/famosab)
 - [Francesco Lescai](https://github.com/lescai)
 - [Gavin Mackenzie](https://github.com/GCJMackenzie)
 - [Gisela Gabernet](https://github.com/ggabernet)
 - [Grant Neilson](https://github.com/grantn5)
 - [gulfshores](https://github.com/gulfshores)
 - [Harshil Patel](https://github.com/drpatelh)
+- [Hongwei Ye](https://github.com/YeHW)
 - [James A. Fellows Yates](https://github.com/jfy133)
 - [Jesper Eisfeldt](https://github.com/J35P312)
 - [Johannes Alneberg](https://github.com/alneberg)

assets/multiqc_config.yml

@@ -3,9 +3,9 @@ custom_logo_url: https://github.com/nf-core/sarek/
 custom_logo_title: "nf-core/sarek"
 
 report_comment: >
-  This report has been generated by the <a href="https://github.com/nf-core/sarek/releases/tag/3.4.2" target="_blank">nf-core/sarek</a>
+  This report has been generated by the <a href="https://github.com/nf-core/sarek/releases/tag/3.4.3" target="_blank">nf-core/sarek</a>
   analysis pipeline. For information about how to interpret these results, please see the
-  <a href="https://nf-co.re/sarek/3.4.2/docs/output" target="_blank">documentation</a>.
+  <a href="https://nf-co.re/sarek/3.4.3/docs/output" target="_blank">documentation</a>.
 report_section_order:
   "nf-core-sarek-methods-description":
     order: -1000