-
Notifications
You must be signed in to change notification settings - Fork 20
230 lines (226 loc) · 8.6 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
name: CI
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
on:
push:
branches:
- dev
pull_request:
release:
types: [published]
env:
NXF_ANSI_LOG: false
# URLs to Influenza ref data should be updated in step with nextflow.config
# default ncbi_influenza_fasta and ncbi_influenza_metadata params
FASTA_ZST_URL: https://api.figshare.com/v2/file/download/41415330
CSV_ZST_URL: https://api.figshare.com/v2/file/download/41415333
concurrency:
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
cancel-in-progress: true
jobs:
test_illumina:
name: Run Illumina test
# Only run on push if this is the nf-flu dev branch (merged PRs)
if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'CFIA-NCFAD/nf-flu') }}
runs-on: ubuntu-latest
env:
NXF_VER: ${{ matrix.nxf_ver }}
NXF_ANSI_LOG: false
strategy:
matrix:
# Nextflow versions: check pipeline minimum and current latest
nxf_ver: ['22.10.1', '23.04.1']
steps:
- name: Check out pipeline code
uses: actions/checkout@v2
- name: Cache Nextflow binary
uses: actions/cache@v3
id: cache-nextflow
with:
path: /usr/local/bin/nextflow
key: nextflow-${{ matrix.nxf_ver }}
- name: Install Nextflow
if: steps.cache-nextflow.outputs.cache-hit != 'true'
env:
CAPSULE_LOG: none
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/
- name: Cache seqtk binary
uses: actions/cache@v3
id: cache-seqtk
with:
path: /usr/local/bin/seqtk
key: seqtk
- name: Build and install seqtk
if: steps.cache-seqtk.outputs.cache-hit != 'true'
run: |
git clone --depth=1 https://github.com/lh3/seqtk.git
cd seqtk
make -j2
make install
which seqtk
- name: Cache subsampled influenza.fna
uses: actions/cache@v3
id: cache-influenza-fna
with:
path: influenza-10k.fna.zst
key: influenza-fna
- name: Subsample NCBI influenza.fna
if: steps.cache-influenza-fna.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${FASTA_ZST_URL} | zstdcat | seqtk sample -s 789 - 10000 | zstd -ck > influenza-10k.fna.zst
- name: Cache influenza.csv
uses: actions/cache@v3
id: cache-influenza-csv
with:
path: influenza.csv.zst
key: influenza-csv
- name: Download influenza.csv
if: steps.cache-influenza-csv.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${CSV_ZST_URL} > influenza.csv.zst
- name: Run pipeline with test data
run: |
nextflow run ${GITHUB_WORKSPACE} \
-profile test_illumina,docker \
--ncbi_influenza_fasta influenza-10k.fna.zst \
--ncbi_influenza_metadata influenza.csv.zst
- name: Upload Artifact
if: success()
uses: actions/upload-artifact@v1.0.0
with:
name: illumina-test-results-${{ matrix.nxf_ver }}
path: results/pipeline_info
- name: Upload .nextflow.log
uses: actions/upload-artifact@v1.0.0
with:
name: nextflow-log-illumina-${{ matrix.nxf_ver }}
path: .nextflow.log
test_nanopore:
name: Run Nanopore test
# Only run on push if this is the nf-flu dev branch (merged PRs)
if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'CFIA-NCFAD/nf-flu') }}
runs-on: ubuntu-latest
env:
NXF_VER: ${{ matrix.nxf_ver }}
NXF_ANSI_LOG: false
strategy:
matrix:
# Nextflow versions: check pipeline minimum and current latest
nxf_ver: ['22.10.1', '23.04.1']
steps:
- name: Check out pipeline code
uses: actions/checkout@v2
- name: Cache Nextflow binary
uses: actions/cache@v3
id: cache-nextflow
with:
path: /usr/local/bin/nextflow
key: nextflow-${{ matrix.nxf_ver }}
- name: Install Nextflow
if: steps.cache-nextflow.outputs.cache-hit != 'true'
env:
CAPSULE_LOG: none
run: |
wget -qO- get.nextflow.io | bash
sudo mv nextflow /usr/local/bin/
- name: Cache seqtk binary
uses: actions/cache@v3
id: cache-seqtk
with:
path: /usr/local/bin/seqtk
key: seqtk
- name: Build and install seqtk
if: steps.cache-seqtk.outputs.cache-hit != 'true'
run: |
git clone --depth=1 https://github.com/lh3/seqtk.git
cd seqtk
make -j2
make install
which seqtk
- name: Cache test sample reads
uses: actions/cache@v3
id: cache-reads
with:
path: reads/
key: reads
- name: Fetch test data
if: steps.cache-reads.outputs.cache-hit != 'true'
run: |
mkdir reads
echo "Downloading ERR6359501 from EBI ENA"
curl -SLk --silent ftp://ftp.sra.ebi.ac.uk/vol1/fastq/ERR635/001/ERR6359501/ERR6359501.fastq.gz > reads/ERR6359501.fastq.gz
- name: Fetch IBV test seq
run: |
curl -SLk --silent https://github.com/CFIA-NCFAD/nf-test-datasets/raw/nf-flu/nanopore/fastq/SRR24826962.sampled.fastq.gz > reads/SRR24826962.fastq.gz
- name: Check IBV data
run: |
file reads/SRR24826962.fastq.gz
md5sum reads/SRR24826962.fastq.gz
sha256sum reads/SRR24826962.fastq.gz
- name: Prepare samplesheet.csv
run: |
echo "Subsample reads from ERR6359501.fastq.gz with seqtk to mock different runs and ways of specifying input"
mkdir run1
seqtk sample -s 1 reads/ERR6359501.fastq.gz 1000 > run1/s1.fq
seqtk sample -s 11 reads/ERR6359501.fastq.gz 1000 | gzip -ck > run1/s11.fastq.gz
mkdir run2
seqtk sample -s 2 reads/ERR6359501.fastq.gz 1000 > run2/s2.fastq
seqtk sample -s 2 reads/ERR6359501.fastq.gz 1000 | gzip -ck > run2/s2.fq.gz
seqtk sample -s 123 reads/ERR6359501.fastq.gz 10000 > reads/ERR6359501-10k.fastq
echo "sample,reads" | tee -a samplesheet.csv
echo "ERR6359501-10k,$(realpath reads/ERR6359501-10k.fastq)" | tee -a samplesheet.csv
echo "ERR6359501,$(realpath run1)" | tee -a samplesheet.csv
echo "ERR6359501,$(realpath run2)" | tee -a samplesheet.csv
echo "SRR24826962,$(realpath reads/SRR24826962.fastq.gz)" | tee -a samplesheet.csv
- name: Cache subsampled influenza.fna
uses: actions/cache@v3
id: cache-influenza-fna
with:
path: influenza-10k.fna.zst
key: influenza-fna
- name: Subsample NCBI influenza.fna
if: steps.cache-influenza-fna.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${FASTA_ZST_URL} | zstdcat | seqtk sample -s 789 - 10000 | zstd -ck > influenza-10k.fna.zst
- name: Cache influenza.csv
uses: actions/cache@v3
id: cache-influenza-csv
with:
path: influenza.csv.zst
key: influenza-csv
- name: Download influenza.csv
if: steps.cache-influenza-csv.outputs.cache-hit != 'true'
run: |
curl --silent -SLk ${CSV_ZST_URL} > influenza.csv.zst
- name: Run pipeline with test data
run: |
nextflow run ${GITHUB_WORKSPACE} \
-profile test_nanopore,docker \
--platform nanopore \
--input samplesheet.csv \
--ncbi_influenza_fasta influenza-10k.fna.zst \
--ncbi_influenza_metadata influenza.csv.zst
- name: Upload pipeline_info/
if: success()
uses: actions/upload-artifact@v1.0.0
with:
name: nanopore-test-results-pipline_info-${{ matrix.nxf_ver }}
path: results/pipeline_info
- name: Upload nf-flu-subtyping-report.xlsx
if: success()
uses: actions/upload-artifact@v1.0.0
with:
name: nanopore-test-results-subtyping-report-${{ matrix.nxf_ver }}
path: results/nf-flu-subtyping-report.xlsx
- name: Upload multiqc_report.html
if: success()
uses: actions/upload-artifact@v1.0.0
with:
name: nanopore-test-results-multiqc-${{ matrix.nxf_ver }}
path: results/MultiQC/multiqc_report.html
- name: Upload .nextflow.log
uses: actions/upload-artifact@v1.0.0
with:
name: nextflow-log-nanopore-${{ matrix.nxf_ver }}
path: .nextflow.log