Skip to content

Commit

Permalink
handle null characters in xargs
Browse files Browse the repository at this point in the history
  • Loading branch information
jethror1 committed Feb 12, 2024
1 parent 87e6e05 commit df04cb7
Showing 1 changed file with 4 additions and 5 deletions.
9 changes: 4 additions & 5 deletions src/code.sh
Original file line number Diff line number Diff line change
Expand Up @@ -348,8 +348,7 @@ _upload_single_file() {
file_id=$(dx upload "$file" --path "$remote_path" --parents --brief)

if [[ "$link" == true ]]; then
# flock here ensures we don't try write to the dxoutput.json when uploading in parallel
flock -x -w30 /tmp/add_output.lock dx-jobutil-add-output "$field" "$file_id" --array
dx-jobutil-add-output "$field" "$file_id" --array
fi
}

Expand Down Expand Up @@ -487,11 +486,11 @@ _upload_final_output() {
xargs -n1 -I{} mv {} /tmp <<< $files
done

# compress intermediate genome VCFs since we don't use these routinely
# and they go from >300mb to < 10mb (plus its a vcf, it should be compressed)
# compress intermediate VCFs since we don't use these routinely and
# they go from >300mb to < 10mb (plus its a vcf, it should be compressed)
time find /home/dnanexus/out/scatter/ /home/dnanexus/out/gather/Logs_Intermediates/ \
-type f -name "*.vcf" -print0 \
| xargs -I{} -n1 -P "$THREADS" gzip {}
| xargs --null -I{} -n1 -P "$THREADS" gzip {}

# upload final run level MetricsOutput.tsv as distinct output field
metrics_file_id=$(dx upload -p /home/dnanexus/out/gather/Results/MetricsOutput.tsv --brief)
Expand Down

0 comments on commit df04cb7

Please sign in to comment.