parallelise final vcf compression - fixes #18

eastgenomics · Feb 12, 2024 · 2781c8c · 2781c8c
1 parent ae56337
commit 2781c8c
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/src/code.sh b/src/code.sh
@@ -488,7 +488,8 @@ _upload_final_output() {
 
     # compress intermediate genome VCFs since we don't use these routinely
     # and they go from >300mb to < 10mb (plus its a vcf, it should be compressed)
-    find "/home/dnanexus/out/scatter/" -type f -name "*.vcf"  -exec gzip {} \;
+    find "/home/dnanexus/out/scatter/" -type f -name "*.vcf"  -print0 \
+        | xargs -I{} -n1 -P "$THREADS" gzip {}
 
     # upload final run level MetricsOutput.tsv as distinct output field
     metrics_file_id=$(dx upload -p /home/dnanexus/out/gather/Results/MetricsOutput.tsv --brief)