Skip to content

Commit

Permalink
Merge branch 'master' into shapeit5_switch
Browse files Browse the repository at this point in the history
  • Loading branch information
LouisLeNezet authored Dec 4, 2024
2 parents d5f2c20 + 8dccae4 commit 4932c5b
Show file tree
Hide file tree
Showing 299 changed files with 12,030 additions and 3,296 deletions.
11 changes: 4 additions & 7 deletions .github/actions/nf-test-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ inputs:
total_shards:
description: "Total number of test shards(NOT the total number of matrix jobs)"
required: true
tags:
description: "Tags to test (`<tags>[,<tags>...]`)"
required: false
paths:
description: "Test paths"
required: true

runs:
using: "composite"
Expand Down Expand Up @@ -72,19 +72,16 @@ runs:
env:
SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
SENTIEON_AUTH_MECH: "GitHub Actions - token"
TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }}
run: |
NFT_WORKDIR=~ \
nf-test test \
--profile=${{ inputs.profile }} \
--tap=test.tap \
--verbose \
--ci \
--changed-since HEAD^ \
--shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
--filter process,workflow \
--follow-dependencies \
${{ env.TAGS }}
${{ inputs.paths }}
# TODO If no test.tap, then make one to spoof?
- uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
Expand Down
43 changes: 27 additions & 16 deletions .github/workflows/gpu-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,15 @@ on:
- "renovate/**" # branches Renovate creates
pull_request:
branches: [master]
paths:
- ".github/workflows/gpu-tests.yml"
- "modules/nf-core/parabricks/**"
merge_group:
types: [checks_requested]
branches: [master]
workflow_dispatch:
inputs:
runners:
description: "Runners to test on"
type: choice
options:
- "ubuntu-latest"
- "self-hosted"
default: "self-hosted"
type: string
default: "gpu"

# Cancel if a newer run is started
concurrency:
Expand All @@ -43,7 +37,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
# Expose detected tags as 'modules' and 'workflows' output variables
paths: ${{ steps.outputs.outputs.components }}
paths: ${{ steps.list.outputs.components }}
modules: ${{ steps.outputs.outputs.modules }}
subworkflows: ${{ steps.outputs.outputs.subworkflows}}
# Prod for version bumping
Expand All @@ -60,11 +54,11 @@ jobs:

- name: List nf-test files
id: list
uses: adamrtalbot/detect-nf-test-changes@6bf6fd9fe0fb63a0362fb0e09de5acb6d055a754 # v0.0.5
uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
with:
head: ${{ github.sha }}
base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
n_parents: 2
n_parents: 0
tags: "gpu"

- name: Separate modules and subworkflows
Expand All @@ -74,21 +68,20 @@ jobs:
echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
- name: debug
run: |
echo ${{ steps.outputs.outputs.components }}
echo ${{ steps.list.outputs.components }}
echo ${{ steps.outputs.outputs.modules }}
echo ${{ steps.outputs.outputs.subworkflows }}
nf-test-gpu:
runs-on: "gpu"
name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}"
needs: nf-test-changes
if: ${{ fromJSON(needs.nf-test-changes.outputs.paths) != '[]' || needs.nf-test-changes.outputs.paths != '' }}
name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}"
if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
strategy:
fail-fast: false
matrix:
shard: [1, 2]
profile: [docker_self_hosted, singularity] # conda?
tags: ${{ fromJSON(needs.nf-test-changes.outputs.modules) && fromJSON(needs.nf-test-changes.outputs.subworkflows) }}
env:
NXF_ANSI_LOG: false
TOTAL_SHARDS: 2
Expand All @@ -108,4 +101,22 @@ jobs:
profile: ${{ matrix.profile }},gpu
shard: ${{ matrix.shard }}
total_shards: ${{ env.TOTAL_SHARDS }}
tags: ${{matrix.tags}},gpu
paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"

confirm-pass:
runs-on: ubuntu-latest
needs: [nf-test-gpu]
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1

- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
66 changes: 6 additions & 60 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@ jobs:
with:
filters: |
modules:
- 'modules/nf-core/**'
- 'tests/modules/nf-core/**'
- added|modified: 'modules/nf-core/**'
- added|modified: 'tests/modules/nf-core/**'
subworkflows:
- 'subworkflows/nf-core/**'
- 'tests/subworkflows/nf-core/**'
- added|modified: 'subworkflows/nf-core/**'
- added|modified: 'tests/subworkflows/nf-core/**'
token: ""
list-files: "json"

Expand All @@ -80,37 +80,17 @@ jobs:
script: |
return [...new Set(${{ steps.filter.outputs.modules_files }}
.map(path => path
.replace('tests/', '')
.replace('modules/nf-core/', '')
.split('/')
.slice(0, 2)
.filter(x => x !== 'main.nf' && x !== 'tests' && x !== 'meta.yml' && x !== 'environment.yml')
.filter(x => !x.startsWith('main.nf') && x !== 'tests' && x !== 'meta.yml' && x !== 'environment.yml')
.join('/'))
)
];
- name: Get subworkflow name
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
id: subworkflow_names
with:
script: |
return [...new Set(${{ steps.filter.outputs.subworkflows_files }}
.map(path => path
.replace('tests/', '')
.replace('subworkflows/nf-core/', '')
.split('/')
.slice(0, 2)
.filter(x => x !== 'main.nf' && x !== 'tests' && x !== 'meta.yml' && x !== 'environment.yml')
.join('/'))
)
];
- name: debug
run: |
echo ${{ steps.filter.outputs.modules_files }}
echo ${{ steps.filter.outputs.subworkflows_files }}
echo ${{ steps.module_names.outputs.result }}
echo ${{ steps.subworkflow_names.outputs.result }}
nf-core-lint-modules:
runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
Expand Down Expand Up @@ -154,43 +134,9 @@ jobs:
- name: Lint module ${{ matrix.module }}
run: nf-core modules lint ${{ matrix.module }}

nf-core-lint-subworkflows:
runs-on: ubuntu-latest
name: nf-core lint subworkflows
needs: nf-core-changes
if: ${{ (needs.nf-core-changes.outputs.subworkflows == 'true') }}
strategy:
fail-fast: false
matrix:
subworkflow: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows_files) }}"
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

- name: Set up Python
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
with:
python-version: "3.11"

- name: Install pip
run: python -m pip install --upgrade pip

- uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
with:
distribution: "temurin"
java-version: "17"

- name: Setup Nextflow
uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2

- name: Install nf-core tools development version
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev

- name: Lint subworkflow ${{ matrix.subworkflow }}
run: nf-core subworkflows lint ${{ matrix.subworkflow }}

confirm-pass:
runs-on: ubuntu-latest
needs: [nf-core-lint-modules, nf-core-lint-subworkflows]
needs: [nf-core-lint-modules]
if: always()
steps:
- name: All tests ok
Expand Down
91 changes: 68 additions & 23 deletions .github/workflows/nf-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,45 +35,90 @@ env:
NXF_VER: "24.10.1"

jobs:
nf-test-changes:
name: nf-test-changes
runs-on: ubuntu-latest
outputs:
# Expose detected tags as 'modules' and 'workflows' output variables
paths: ${{ steps.list.outputs.components }}
modules: ${{ steps.outputs.outputs.modules }}
subworkflows: ${{ steps.outputs.outputs.subworkflows}}
# Prod for version bumping
steps:
- name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
fetch-depth: 0

- name: List nf-test files
id: list
uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
with:
head: ${{ github.sha }}
base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
n_parents: 0
exclude_tags: "gpu"

- name: Separate modules and subworkflows
id: outputs
run: |
echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
- name: debug
run: |
echo ${{ steps.list.outputs.components }}
echo ${{ steps.outputs.outputs.modules }}
echo ${{ steps.outputs.outputs.subworkflows }}
nf-test:
runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
# NOTE I think this is the cleanest way to get them organized
# process | conda | 1
# process | conda | 2
# process | conda | 3
# process | docker_self_hosted | 1
# ...
# workflow | singularity | 3
name: "${{ matrix.profile }} | ${{ matrix.shard }}"
# TODO
# needs: get-number-of-shards
# if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }}
needs: nf-test-changes
if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
strategy:
fail-fast: false
matrix:
# NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now
# NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}"
# filter: [process, workflow]
profile: [conda, docker_self_hosted, singularity]
shard: [1, 2, 3, 4, 5]
profile: [conda, docker_self_hosted, singularity]
env:
# FIXME Bumping them up to make the transition smooth, then we can throttle them back
NXF_ANSI_LOG: false
TOTAL_SHARDS: 5
SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}

steps:
- name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
fetch-depth: 0
- name: Run nf-test Action
uses: ./.github/actions/nf-test-action
env:
SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
SENTIEON_AUTH_MECH: "GitHub Actions - token"
with:
profile: ${{ matrix.profile }}
shard: ${{ matrix.shard }}
total_shards: ${{ env.TOTAL_SHARDS }}
paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"

confirm-pass:
runs-on: ubuntu-latest
needs: [nf-test]
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1

- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
4 changes: 4 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,7 @@ repos:
hooks:
- id: editorconfig-checker
alias: ec
- repo: https://github.com/nf-core/tools
rev: pre-commit
hooks:
- id: lint-subworkflows
Loading

0 comments on commit 4932c5b

Please sign in to comment.