EVA-2637 EVA submission docker environment (#154)

eva submission docker environment

eva-submission-docker/docker-compose.yml

@@ -0,0 +1,60 @@
+version: "3.8"
+
+services:
+  eva_submission:
+    image: eva_submission
+    build:
+      context: ./
+      dockerfile: eva_docker/Dockerfile
+    container_name: eva_submission
+    command: tail -f /dev/null
+    depends_on:
+      - eva-ws
+      - postgres_db
+      - mongo_db
+    networks:
+      - eva_network
+
+  eva-ws:
+    image: eva-ws
+    build:
+      context: ./
+      dockerfile: ws_docker/Dockerfile
+    container_name: eva-ws
+    restart: always
+    ports:
+      - 8080:8080
+    depends_on:
+      - postgres_db
+    networks:
+      - eva_network
+
+  postgres_db:
+    image: postgres_db
+    build: postgres_docker/.
+    container_name: postgres_db
+    restart: always
+    environment:
+      - POSTGRES_USER=root_user
+      - POSTGRES_PASSWORD=root_pass
+    ports:
+      - 5432:5432
+    networks:
+      - eva_network
+
+  mongo_db:
+    image: mongo_db
+    build: mongo_docker/.
+    container_name: mongo_db
+    command: tail -f /dev/null
+    restart: always
+    ports:
+      - 27017:27017
+      - 27018:27018
+      - 27019:27019
+    networks:
+      - eva_network
+
+networks:
+  eva_network:
+    name: eva_network

eva-submission-docker/eva_docker/Dockerfile

@@ -0,0 +1,110 @@
+FROM ubuntu:22.04
+
+ENV JAVA_VERSION 8
+ENV NXF_VER 21.10.5
+ENV VCF_VALIDATOR_VERSION 0.9.4
+ENV MONGO_VERSION 4.4
+ENV EVA_ACCESSION_VERSION v0.6.21
+ENV EVA_PIPELINE_VERSION v2.9.1
+ENV EVA_SUBMISSION_VERSION v1.10.6
+ENV VARIANT_REMAPPING v1.0.4
+
+COPY maven-settings.xml /root
+COPY eva_docker/submission_config.yml /root/.submission_config.yml
+
+RUN mkdir -p /usr/local/test_eva_submission/data
+RUN mkdir -p /usr/local/test_eva_submission/submissions
+RUN mkdir -p /usr/local/eva/public_ftp/
+RUN mkdir -p /usr/local/eva/datasources/reference_sequences
+RUN mkdir -p /usr/local/eva/datasources/vep-cache
+RUN mkdir -p /usr/local/software/opencga
+
+ADD eva_docker/ftp /usr/local/eva/ftp/
+ADD eva_docker/conf /usr/local/software/opencga/conf
+
+# install utility libs
+RUN apt update  \
+    && apt install -y curl wget maven git vim samtools bcftools tabix bedtools minimap2 rsync  \
+    && apt install -y python3-pip python3.10-venv
+
+# install java
+RUN apt install -y openjdk-${JAVA_VERSION}-jdk
+# Set Java Home
+ENV JAVA_HOME /usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64
+ENV PATH $JAVA_HOME/bin:$PATH
+
+# install nextflow
+RUN curl -L "https://github.com/nextflow-io/nextflow/releases/download/v${NXF_VER}/nextflow-${NXF_VER}-all" | bash \
+    && mv nextflow /usr/local/bin/
+
+# install mongodb tools
+RUN wget -qO - https://www.mongodb.org/static/pgp/server-${MONGO_VERSION}.asc | apt-key add -
+RUN echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${MONGO_VERSION} multiverse" | tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list
+RUN apt update && apt install -y mongodb-database-tools
+
+# install mongodb shell
+RUN wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+RUN dpkg -i ./libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+RUN apt install -y mongodb-org-shell
+
+WORKDIR /usr/local/software
+
+COPY eva_docker/mocked_load_from_ena_postgres_or_file.pl /usr/local/software
+
+# install vcf_validator
+RUN curl -LJo /usr/local/software/vcf_validator https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_validator_linux \
+    && chmod 755 /usr/local/software/vcf_validator
+
+# install vcf_assembly_checker
+RUN curl -LJo /usr/local/software/vcf_assembly_checker https://github.com/EBIvariation/vcf-validator/releases/download/v${VCF_VALIDATOR_VERSION}/vcf_assembly_checker_linux \
+    && chmod 755 /usr/local/software/vcf_assembly_checker
+
+# setup eva-accession
+RUN git clone -b ${EVA_ACCESSION_VERSION} https://github.com/EBIvariation/eva-accession.git
+RUN mvn package -s /root/maven-settings.xml -f eva-accession/pom.xml -DskipTests -P docker
+RUN cp eva-accession/eva-accession-pipeline/target/eva-accession-pipeline-*-exec.jar /usr/local/software/eva-accession-pipeline.jar
+RUN cp eva-accession/eva-accession-clustering/target/eva-accession-clustering-*-exec.jar /usr/local/software/eva-accession-clustering.jar
+RUN cp eva-accession/eva-remapping-get-source/target/eva-remapping-get-source-*-exec.jar /usr/local/software/eva-remapping-get-source.jar
+RUN cp eva-accession/eva-remapping-ingest/target/eva-remapping-ingest-*-exec.jar /usr/local/software/eva-remapping-ingest.jar
+
+# setup eva-pipeline
+RUN git clone -b ${EVA_PIPELINE_VERSION} https://github.com/EBIvariation/eva-pipeline.git
+RUN mvn package -s /root/maven-settings.xml -f eva-pipeline/pom.xml -DskipTests -P docker
+RUN cp eva-pipeline/target/eva-pipeline-*.jar /usr/local/software/eva-pipeline.jar
+
+# setup eva-submission
+RUN git clone -b ${EVA_SUBMISSION_VERSION} https://github.com/EBIvariation/eva-submission.git
+RUN python3 -m venv eva-submission/venv
+RUN eva-submission/venv/bin/pip3 install --upgrade pip
+RUN eva-submission/venv/bin/pip3 install /usr/local/software/eva-submission
+
+# setup variant-remapping
+RUN git clone -b ${VARIANT_REMAPPING} https://github.com/EBIvariation/variant-remapping.git
+RUN python3 -m venv variant-remapping/venv
+RUN variant-remapping/venv/bin/pip3 install --upgrade pip
+RUN variant-remapping/venv/bin/pip3 install -r variant-remapping/requirements.txt
+
+ENV PATH="${PATH}:/usr/local/software/eva-submission/venv/bin"
+
+# install VEP
+WORKDIR /usr/local/software/vep
+
+# install general dependencies for vep
+RUN apt install -y tar libdbi-perl libdbd-mysql-perl libbz2-dev liblzma-dev
+
+# install htslib
+RUN wget https://github.com/samtools/htslib/releases/download/1.9/htslib-1.9.tar.bz2 -P /usr/bin
+RUN tar -vxjf /usr/bin/htslib-1.9.tar.bz2
+RUN make -C /usr/bin/htslib-1.9/
+ENV PATH="${PATH}:/usr/bin/htslib-1.9"
+
+# install perl dependencies for vep
+RUN cpan Alien::Libxml2
+RUN cpan XML::LibXML
+RUN cpan XML::LibXML::Reader
+RUN cpan Bio::SeqFeature::Lite
+RUN cpan Bio::DB::HTS --htslib /usr/bin/htslib-1.9
+
+# install vep
+RUN git clone https://github.com/Ensembl/ensembl-vep.git
+RUN perl ensembl-vep/INSTALL.p

eva-submission-docker/eva_docker/conf/storage-mongodb.properties

@@ -0,0 +1,29 @@
+####
+# Properties file for OpenCGA-Storage-Mongo plugin
+####
+
+#Variant
+## The next properties should match their corresponding ones in application.properties
+## Same as spring.data.mongodb.host
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.HOSTS = mongo_db:27017
+
+## Same as spring.data.mongodb.username
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.USER = mongo_db_user
+
+## Same as spring.data.mongodb.password
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.PASS = mongo_db_pass
+
+## Same as spring.data.mongodb.authentication-database
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.AUTHENTICATION.DB = admin
+
+
+## The next properties should match their corresponding ones in your job properties
+## Same as spring.data.mongodb.database
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.NAME = variants_2_0
+
+## Same as db.collections.variants.name
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.COLLECTION.VARIANTS = variants_2_0
+
+## Same as db.collections.files.name
+OPENCGA.STORAGE.MONGODB.VARIANT.DB.COLLECTION.FILES = files_2_0
+

eva-submission-docker/eva_docker/conf/storage.properties

@@ -0,0 +1,9 @@
+#############################################
+#       STORAGE PARAMETERS                  #
+#############################################
+
+OPENCGA.STORAGE.ENGINES                     = mongodb
+
+## storage-mongodb plugin configuration
+OPENCGA.STORAGE.MONGODB.VARIANT.MANAGER     = org.opencb.opencga.storage.mongodb.variant.MongoDBVariantStorageManager
+OPENCGA.STORAGE.MONGODB.CONF                = storage-mongodb.properties