Merge pull request #467 from nextstrain/surface-nextclade-versions

Surface Nextclade versions

bin/generate-nextclade-version-json

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -euo pipefail
+
+vendored="$(dirname "$0")"/../vendored
+
+
+nextclade="${1:?A path to the Nextclade executable is required as the first argument}"
+nextclade_dataset="${2:?A path to the Nextclade dataset is required as the second argument}"
+nextclade_tsv="${3:?A path to the Nextclade TSV is required as the third argument}"
+
+
+nextclade_version="$("$nextclade" --version)"
+dataset_pathogen_json="$(unzip -p "$nextclade_dataset" pathogen.json)"
+dataset_name="$(echo "$dataset_pathogen_json" | jq -r '.attributes.name')"
+dataset_version="$(echo "$dataset_pathogen_json" | jq -r '.version.tag')"
+nextclade_tsv_sha256sum="$("$vendored/sha256sum" < "$nextclade_tsv")"
+
+jq -c --null-input \
+ --arg NEXTCLADE_VERSION "$nextclade_version" \
+ --arg DATASET_NAME "$dataset_name" \
+ --arg DATASET_VERSION "$dataset_version" \
+ --arg NEXTCLADE_TSV_SHA256SUM "$nextclade_tsv_sha256sum" \
+ '{
+ "schema_version": "v1",
+ "nextclade_version": $NEXTCLADE_VERSION,
+ "nextclade_dataset_name": $DATASET_NAME,
+ "nextclade_dataset_version": $DATASET_VERSION,
+ "nextclade_tsv_sha256sum": $NEXTCLADE_TSV_SHA256SUM
+ }'

workflow/snakemake_rules/nextclade.smk

@@ -316,6 +316,26 @@ rule nextclade_info:
  """
 
 
+rule nextclade_version_json:
+ """
+ Generates a version JSON for the Nextclade TSV.
+ """
+ input:
+ nextclade_path="data/nextclade",
+ nextclade_dataset=lambda w: f"data/nextclade_data/sars-cov-2{w.reference.replace('_','-')}.zip",
+ nextclade_tsv=f"data/{database}/nextclade{{reference}}.tsv",
+ output:
+ nextclade_version_json=f"data/{database}/nextclade{{reference}}_version.json",
+ shell:
+ """
+ ./bin/generate-nextclade-version-json \
+ {input.nextclade_path} \
+ {input.nextclade_dataset} \
+ {input.nextclade_tsv} \
+ > {output.nextclade_version_json}
+ """
+
+
 rule combine_alignments:
  """
  Generating full alignment by combining newly aligned sequences with previous (cached) alignment
@@ -365,3 +385,27 @@ rule generate_metadata:
  --clade-legacy-mapping {input.clade_legacy_mapping} \
  -o {output.metadata}
  """
+
+
+rule metadata_version_json:
+ """
+ Generates the metadata version JSON by adding the metadata TSV sha256sum
+ to the Nextclade version JSON.
+
+ TODO: Merge the 21L Nextclade version JSON to track data provenence for
+ specific columns
+ """
+ input:
+ metadata=f"data/{database}/metadata.tsv",
+ nextclade_version_json=f"data/{database}/nextclade_version.json",
+ output:
+ metadata_version_json=f"data/{database}/metadata_version.json",
+ shell:
+ """
+ metadata_tsv_sha256sum="$(./vendored/sha256sum < {input.metadata})"
+
+ cat {input.nextclade_version_json} \
+ | jq -c --arg METADATA_TSV_SHA256SUM "$metadata_tsv_sha256sum" \
+ '.metadata_tsv_sha256sum = $METADATA_TSV_SHA256SUM' \
+ > {output.metadata_version_json}
+ """

workflow/snakemake_rules/upload.smk

@@ -33,6 +33,9 @@ def compute_files_to_upload():
  "aligned.fasta.zst": f"data/{database}/aligned.fasta",
  "nextclade_21L.tsv.zst": f"data/{database}/nextclade_21L.tsv",
 
+ "nextclade_version.json": f"data/{database}/nextclade_version.json",
+ "nextclade_21L_version.json": f"data/{database}/nextclade_21L_version.json",
+ "metadata_version.json": f"data/{database}/metadata_version.json",
  }
  files_to_upload = files_to_upload | {
  f"translation_{gene}.fasta.zst" : f"data/{database}/translation_{gene}.fasta"