From 23d30f3856c35a9cb1b168f0a79f18d443406fc4 Mon Sep 17 00:00:00 2001 From: April Shen Date: Tue, 13 Feb 2024 16:23:23 +0000 Subject: [PATCH 1/5] WIP - conda recipe --- conda/build.sh | 39 +++++++++++++++++++++++++++++++++++++++ conda/meta.yaml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 conda/build.sh create mode 100644 conda/meta.yaml diff --git a/conda/build.sh b/conda/build.sh new file mode 100644 index 0000000..1390aad --- /dev/null +++ b/conda/build.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +VCF_VALIDATOR_VERSION=0.9.4 +BIOVALIDATOR_VERSION=2.1.0 + +EVA_SUB_CLI="${PREFIX}/share/${PKG_NAME}-${PKG_VERSION}" +mkdir -p ${PREFIX}/bin ${EVA_SUB_CLI} + +# Install eva-sub-cli +$PYTHON -m pip install -r requirements.txt +$PYTHON -m pip install . +cp bin/* ${PREFIX}/bin +echo "Done with eva-sub-cli" + +cd ${EVA_SUB_CLI} + +# Install biovalidator from source +# Includes some workarounds that can be cleaned up once a new version is released +curl -Lo biovalidator.zip https://github.com/elixir-europe/biovalidator/archive/refs/tags/v${BIOVALIDATOR_VERSION}.zip \ + && unzip biovalidator.zip && rm biovalidator.zip \ + && cd biovalidator-${BIOVALIDATOR_VERSION} \ + && bash -c "cat <(echo '#!/usr/bin/env node') <(cat src/biovalidator.js) > tmp" \ + && mv tmp src/biovalidator.js \ + && chmod +x src/biovalidator.js \ + && sed -i 's/dist/src/' package.json \ + && npm install && npm install -g \ + && cd .. +echo "Done with biovalidator" + +# Download pre-built vcf-validator +# Check if linux or osx +if [ -z ${OSX_ARCH+x} ]; then + curl -LJo ${PREFIX}/bin/vcf_validator https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_validator_linux \ + && curl -LJo ${PREFIX}/bin/vcf_assembly_checker https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_assembly_checker_linux \ + && chmod 755 ${PREFIX}/bin/vcf_assembly_checker ${PREFIX}/bin/vcf_validator +else + echo "VCF validator MacOS build not currently functional" +fi +echo "Done with vcf-validator" diff --git a/conda/meta.yaml b/conda/meta.yaml new file mode 100644 index 0000000..83a15ca --- /dev/null +++ b/conda/meta.yaml @@ -0,0 +1,47 @@ +{% set name = "eva-sub-cli" %} +{% set version = "0.1" %} + +package: + name: {{ name }} + version: {{ version }} + +source: + path: ../ +# url: https://github.com/EBIvariation/eva-sub-cli/archive/v{{version}}.tar.gz +# sha256: + +build: + number: 0 + noarch: generic + preserve_egg_dir: True +# run_exports: +# - {{ pin_subpackage(name, max_pin="x.x") }} + +requirements: + host: + - nextflow >=21.10.0 + - python >=3.8 + - pip + # biovalidator + - nodejs >=10.19.1 + run: + - nextflow >=21.10.0 + - python >=3.8 + +test: + imports: + - eva_sub_cli + commands: + - biovalidator --help + - vcf_validator --help + - eva-sub-cli.py --help + +about: + home: https://github.com/EBIvariation/eva-sub-cli + summary: EVA Submission Command Line Interface + license: Apache-2.0 + license_file: LICENSE + +extra: + recipe-maintainers: + - apriltuesday \ No newline at end of file From 0bfc13b3e66d12f37186f5c165b9e10f8e65525b Mon Sep 17 00:00:00 2001 From: April Shen Date: Wed, 14 Feb 2024 13:15:39 +0000 Subject: [PATCH 2/5] conda working with docker already installed and no minify-html --- conda/build.sh | 9 ++++++++- conda/meta.yaml | 12 ++++++++++-- eva_sub_cli/report.py | 8 ++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/conda/build.sh b/conda/build.sh index 1390aad..fdb9a1e 100644 --- a/conda/build.sh +++ b/conda/build.sh @@ -2,18 +2,25 @@ VCF_VALIDATOR_VERSION=0.9.4 BIOVALIDATOR_VERSION=2.1.0 +EVA_PYUTILS_VERSION=0.6.1 EVA_SUB_CLI="${PREFIX}/share/${PKG_NAME}-${PKG_VERSION}" mkdir -p ${PREFIX}/bin ${EVA_SUB_CLI} # Install eva-sub-cli -$PYTHON -m pip install -r requirements.txt $PYTHON -m pip install . cp bin/* ${PREFIX}/bin echo "Done with eva-sub-cli" cd ${EVA_SUB_CLI} +# Install python dependencies not yet on conda +curl -Lo eva-pyutils.zip https://github.com/EBIvariation/eva-common-pyutils/archive/refs/tags/v${EVA_PYUTILS_VERSION}.zip \ + && unzip eva-pyutils.zip && rm eva-pyutils.zip \ + && cd eva-common-pyutils-${EVA_PYUTILS_VERSION} \ + && $PYTHON setup.py install \ + && cd .. + # Install biovalidator from source # Includes some workarounds that can be cleaned up once a new version is released curl -Lo biovalidator.zip https://github.com/elixir-europe/biovalidator/archive/refs/tags/v${BIOVALIDATOR_VERSION}.zip \ diff --git a/conda/meta.yaml b/conda/meta.yaml index 83a15ca..4a9b331 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -21,12 +21,20 @@ requirements: host: - nextflow >=21.10.0 - python >=3.8 - - pip - # biovalidator - nodejs >=10.19.1 + - pyyaml + - jinja2 + - openpyxl + - requests + - jsonschema run: - nextflow >=21.10.0 - python >=3.8 + - pyyaml + - jinja2 + - openpyxl + - requests + - jsonschema test: imports: diff --git a/eva_sub_cli/report.py b/eva_sub_cli/report.py index ec313d3..95c0b57 100644 --- a/eva_sub_cli/report.py +++ b/eva_sub_cli/report.py @@ -2,7 +2,6 @@ import os.path from jinja2 import Environment, FileSystemLoader -from minify_html import minify_html current_dir = os.path.dirname(__file__) @@ -30,4 +29,9 @@ def generate_html_report(validation_results, validation_date, project_title=None validation_results=validation_results, ) - return minify_html.minify(rendered_template, minify_js=True, remove_processing_instructions=True) + try: + # minify-html is not included in conda installation currently + from minify_html import minify_html + return minify_html.minify(rendered_template, minify_js=True, remove_processing_instructions=True) + except ImportError: + return rendered_template From bbf9b8a27005b492386f3f56884ca445bfc982cb Mon Sep 17 00:00:00 2001 From: April Shen Date: Fri, 16 Feb 2024 13:45:38 +0000 Subject: [PATCH 3/5] bump version and add macOS vcf-validator --- conda/build.sh | 6 ++++-- conda/meta.yaml | 2 +- eva_sub_cli/VERSION | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/conda/build.sh b/conda/build.sh index fdb9a1e..5921457 100644 --- a/conda/build.sh +++ b/conda/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -VCF_VALIDATOR_VERSION=0.9.4 +VCF_VALIDATOR_VERSION=0.9.6 BIOVALIDATOR_VERSION=2.1.0 EVA_PYUTILS_VERSION=0.6.1 @@ -41,6 +41,8 @@ if [ -z ${OSX_ARCH+x} ]; then && curl -LJo ${PREFIX}/bin/vcf_assembly_checker https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_assembly_checker_linux \ && chmod 755 ${PREFIX}/bin/vcf_assembly_checker ${PREFIX}/bin/vcf_validator else - echo "VCF validator MacOS build not currently functional" + curl -LJo ${PREFIX}/bin/vcf_validator https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_validator_macos \ + && curl -LJo ${PREFIX}/bin/vcf_assembly_checker https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_assembly_checker_macos \ + && chmod 755 ${PREFIX}/bin/vcf_assembly_checker ${PREFIX}/bin/vcf_validator fi echo "Done with vcf-validator" diff --git a/conda/meta.yaml b/conda/meta.yaml index 4a9b331..ab1b51d 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "eva-sub-cli" %} -{% set version = "0.1" %} +{% set version = "0.2" %} package: name: {{ name }} diff --git a/eva_sub_cli/VERSION b/eva_sub_cli/VERSION index ceab6e1..2f45361 100644 --- a/eva_sub_cli/VERSION +++ b/eva_sub_cli/VERSION @@ -1 +1 @@ -0.1 \ No newline at end of file +0.2 \ No newline at end of file From 6c2fdb2f1e066de8c208b5a642acb4f1a3bf47b6 Mon Sep 17 00:00:00 2001 From: April Shen Date: Fri, 16 Feb 2024 14:11:04 +0000 Subject: [PATCH 4/5] add installation documentation --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- bin/eva-sub-cli.py | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e7a21b..ad36b12 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,48 @@ EVA Submission Command Line Interface for Validation ## Installation -TBD +There are currently three ways to install and run the tool: using conda, from source using Docker, +and from source natively (i.e. managing dependencies on your own). + +### Conda + +The most straightforward way to install eva-sub-cli and its dependencies is through conda. +For example the following installs eva-sub-cli in a new environment called `eva`, activates the environment, and prints +the help message: +```bash +conda create -n eva -c conda-forge -c bioconda eva-sub-cli +conda activate eva +eva-sub-cli.py --help +```` + +### From source using Docker + +This method requires just Python 3.8+ and [Docker](https://docs.docker.com/engine/install/) to be installed. +Then either clone the git repository, or download the newest tagged release from [here](https://github.com/EBIvariation/eva-sub-cli/tags): +```bash +git clone git@github.com:EBIvariation/eva-sub-cli.git +# OR +wget -O eva-sub-cli.zip https://github.com/EBIvariation/eva-sub-cli/archive/refs/tags/v0.2.zip +unzip eva-sub-cli.zip +``` + +Then install the library and its dependencies as follows (e.g. in a virtual environment): +```bash +cd eva-sub-cli +pip install -r requirements.txt +python setup.py install +``` + +### From source natively + +This method requires the following: +* Python 3.8+ +* [Nextflow](https://www.nextflow.io/docs/latest/getstarted.html) 21.10+ +* [biovalidator](https://github.com/elixir-europe/biovalidator) 2.1.0+ +* [vcf-validator](https://github.com/EBIvariation/vcf-validator) 0.9.6+ + +Install each of these and ensure they are available on the path. +Then git clone the repo or install the newest release as described above. ## Input files for the validation and submission tool @@ -46,6 +87,9 @@ More detail documentation to follow ## Execution +Note for Docker users: for each of the below commands, add the command line option `--executor docker`, which will +fetch and manage the docker container for you. + ### Validate and submit your dataset To validate and submit run the following command diff --git a/bin/eva-sub-cli.py b/bin/eva-sub-cli.py index 1008328..a4de0b5 100755 --- a/bin/eva-sub-cli.py +++ b/bin/eva-sub-cli.py @@ -55,7 +55,7 @@ def validate_command_line_arguments(args, argparser): 'previous runs. Selecting SUBMIT will run validate only if the validation was not performed ' 'successfully before and then run the submission.') argparser.add_argument('--executor', choices=[DOCKER, NATIVE], default=NATIVE, - help='Select an execution type for running validation') + help='Select an execution type for running validation (default native)') credential_group = argparser.add_argument_group('Credential', 'Specify the Webin credential you want to use to ' 'upload to the EVA') credential_group.add_argument("--username", help="Username used for connecting to the ENA webin account") From 312d67355c6acb147eba9f14f7d6bf85e52e441b Mon Sep 17 00:00:00 2001 From: April Shen Date: Tue, 20 Feb 2024 15:06:27 +0000 Subject: [PATCH 5/5] add comments to meta --- conda/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/meta.yaml b/conda/meta.yaml index ab1b51d..40ba37e 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -7,6 +7,7 @@ package: source: path: ../ +# For submission to bioconda recipes, replace 'path' with the following (filling in the appropriate sha256): # url: https://github.com/EBIvariation/eva-sub-cli/archive/v{{version}}.tar.gz # sha256: @@ -14,6 +15,7 @@ build: number: 0 noarch: generic preserve_egg_dir: True +# For submission to bioconda recipes, replace 'preserve_egg_dir' with the following: # run_exports: # - {{ pin_subpackage(name, max_pin="x.x") }}