Skip to content

Commit

Permalink
Merge pull request #26 from bento-platform/qa/v3.5
Browse files Browse the repository at this point in the history 
v3.5.3 rc-1
  • Loading branch information
@brouillette
brouillette authored Sep 16, 2022
2 parents b91d6f8 + 9a3fb50 commit 037ffa6
Show file tree
Hide file tree
Showing 8 changed files with 279 additions and 173 deletions.
76 changes: 40 additions & 36 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ SHELL = bash
include $(env)
export $(shell sed 's/=.*//' $(env))

export GOOS=linux
export GOARCH=amd64

# export host user IDs for more secure
# containerization and volume mounting
export HOST_USER_UID=$(shell id -u)
Expand All @@ -36,41 +33,43 @@ init:

@$(MAKE) init-data-dirs
@$(MAKE) init-vendor
@$(MAKE) init-networks


init-networks:
docker network create ${GOHAN_DOCKER_NET} &

init-vendor:
@echo "Initializing Go Module Vendor"
@cd src/api && go mod vendor && \
cd ../tests && go mod vendor
cd src/api && go mod vendor
cd src/tests && go mod tidy && go mod vendor

init-data-dirs:
@echo "Initializing data directories.." && \
# api-drs bridge: \
mkdir -p ${GOHAN_API_DRS_BRIDGE_HOST_DIR} && \
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_DRS_BRIDGE_HOST_DIR} && \
chmod -R 770 ${GOHAN_API_DRS_BRIDGE_HOST_DIR} && \
\
# drs: \
mkdir -p ${GOHAN_DRS_DATA_DIR} && \
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_DRS_DATA_DIR} && \
chmod -R 770 ${GOHAN_DRS_DATA_DIR} && \
\
# elasticsearch: \
mkdir -p ${GOHAN_ES_DATA_DIR} && \
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_ES_DATA_DIR} && \
chmod -R 770 ${GOHAN_ES_DATA_DIR} && \
chmod -R 770 ${GOHAN_ES_DATA_DIR} && \
\
# tmp: \
# (setup for when gohan needs to preprocess vcf's at ingestion time): \
mkdir -p ${GOHAN_API_VCF_PATH}/tmp && \
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_VCF_PATH}/tmp && \
chmod -R 770 ${GOHAN_API_VCF_PATH}/tmp && \
chmod -R 770 ${GOHAN_API_VCF_PATH}/tmp && \
mkdir -p ${GOHAN_API_GTF_PATH} && \
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_GTF_PATH} && \
chmod -R 770 ${GOHAN_API_GTF_PATH}/ && \
chmod -R 770 ${GOHAN_API_GTF_PATH}/tmp && \
echo ".. done!"
mkdir ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
chmod -R 770 ${GOHAN_API_DRS_BRIDGE_HOST_DIR}

mkdir ${GOHAN_DRS_DATA_DIR}
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_DRS_DATA_DIR}
chmod -R 770 ${GOHAN_DRS_DATA_DIR}

mkdir ${GOHAN_ES_DATA_DIR}
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_ES_DATA_DIR}
chmod -R 770 ${GOHAN_ES_DATA_DIR}

@# tmp:
@# (setup for when gohan needs to preprocess vcf's at ingestion time):
mkdir ${GOHAN_API_VCF_PATH}
mkdir -p ${GOHAN_API_VCF_PATH}/tmp
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_VCF_PATH}
chmod -R 770 ${GOHAN_API_VCF_PATH}/tmp

mkdir ${GOHAN_API_GTF_PATH}
mkdir -p ${GOHAN_API_GTF_PATH}/tmp
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_GTF_PATH}
chmod -R 770 ${GOHAN_API_GTF_PATH}/tmp

@echo ".. done!"


# Run
Expand Down Expand Up @@ -101,8 +100,10 @@ build-api-local-binaries:
\
go build -ldflags="-s -w" -o ../../bin/api_${GOOS}_${GOARCH} && \
\
cd ../.. && \
upx --brute bin/api_${GOOS}_${GOARCH}
cd ../..
# Temporarily disabling:
# && \
# upx --brute bin/api_${GOOS}_${GOARCH}

build-api: stop-api clean-api
@echo "-- Building Golang-Api-Alpine Container --"
Expand All @@ -127,7 +128,10 @@ stop-%:


# Clean up
clean-all: clean-api clean-gateway clean-drs
clean-all: clean-networks clean-api clean-gateway clean-drs

clean-networks:
docker network remove ${GOHAN_DOCKER_NET} &

clean-gateway:
docker rm ${GOHAN_GATEWAY_CONTAINER_NAME} --force; \
Expand Down
211 changes: 132 additions & 79 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@


## Prerequisites
- Golang >= 1.15.5
- Golang >= 1.19
- installation: https://golang.org/doc/install
- UPX
- docs: https://github.com/upx/upx
- other references
- https://linguinecode.com/post/install-golang-linux-terminal
- Elasticsearch
- getting started: https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started.html
- overview tutorial: https://www.youtube.com/watch?v=C3tlMqaNSaI
Expand All @@ -21,16 +21,87 @@
- overview: https://en.wikipedia.org/wiki/Make_(software)
- Docker
- getting started: https://www.docker.com/get-started
- other references:
- linux installation: https://www.tutorialspoint.com/docker/installing_docker_on_linux.htm
- linux running without sudo : https://dev.to/nabbisen/docker-without-sudo-34ci
- Docker-Compose
- getting started: https://docs.docker.com/compose/gettingstarted/
- Visual Studio Code (recommended)
- getting started: https://code.visualstudio.com/docs
- Golang extension: https://marketplace.visualstudio.com/items?itemName=golang.go
- PERL (optional)
- installation: https://learn.perl.org/installing/unix_linux.html

- htspwd
- linux installation: https://www.digitalocean.com/community/tutorials/how-to-set-up-password-authentication-with-apache-on-ubuntu-18-04-quickstart
<br />


## TL;DR

### Typical use-case walkthrough
```
# environment
cp ./etc/example.env .env # modify to your needs
# kickstart
make init
# gateway & certificates
mkdir -p gateway/certs/dev
openssl req -newkey rsa:2048 -nodes -keyout gateway/certs/dev/gohan_privkey1.key -x509 -days 365 -out gateway/certs/dev/gohan_fullchain1.crt
openssl req -newkey rsa:2048 -nodes -keyout gateway/certs/dev/es_gohan_privkey1.key -x509 -days 365 -out gateway/certs/dev/es_gohan_fullchain1.crt
make build-gateway && make run-gateway
# elasticsearch
make run-elasticsearch
# services
make build-drs && make run-drs
make build-api && make run-api
# initiate genes catlogue:
curl -k https://gohan.local/genes/ingestion/run
# monitor progress:
curl -k https://gohan.local/genes/ingestion/requests
curl -k https://gohan.local/genes/ingestion/stats
# view catalogue
curl -k https://gohan.local/genes/overview
# create table
DATA='{
"name": "Gohan Box Test Table",
"data_type": "variant",
"dataset": "00000000-0000-0000-0000-000000000000",
"metadata": {}
}'
curl -k -0 -v -X POST https://gohan.local/tables \
-H 'Content-Type:application/json' \
--data "$(echo $DATA)" | jq
# <obtain the table "id">
# move vcf.gz files to `$GOHAN_API_VCF_PATH`
# ingest vcf.gz
curl -k https://gohan.local/variants/ingestion/run\?fileNames=<filename>\&assemblyId=GRCh37\&filterOutHomozygousReferences=true\&tableId=<table id>
# monitor progress:
curl -k https://gohan.local/variants/ingestion/requests
curl -k https://gohan.local/variants/ingestion/stats
# view variants
curl -k https://gohan.local/variants/overview
```

## Getting started

### **Environment :**
Expand All @@ -41,10 +112,11 @@ First, from the project root, create a local file for environment variables with
cp ./etc/example.env .env
```
and make any necessary changes, such as the Elasticsearch `GOHAN_ES_USERNAME` and `GOHAN_ES_PASSWORD` when in production.
> note: a known current bug is that GOHAN_ES_USERNAME must remain its default..
<br >

### **Init**
### **Initialization**
Run
```
make init
Expand Down Expand Up @@ -117,37 +189,72 @@ make run-gateway
```

<br />
<br />


### **API**
<b>Containerized : </b>

From the project root, run
&nbsp;To simply run a working instance of the api "out of the box", build the `docker image` and spawn the `container` with an fresh binary build by running

```
make build-api
make run-api
```
export GOHAN_API_INTERNAL_PORT=${GOHAN_API_INTERNAL_PORT}
export GOHAN_API_VCF_PATH=${GOHAN_API_VCF_PATH}

# Elasticsearch
export GOHAN_ES_URL=${GOHAN_PRIVATE_ES_URL}
export GOHAN_ES_USERNAME=${GOHAN_ES_USERNAME}
export GOHAN_ES_PASSWORD=${GOHAN_ES_PASSWORD}
&nbsp;and the `docker-compose.yaml` file will handle the configuration.


# AuthX
export GOHAN_AUTHZ_ENABLED=${GOHAN_API_AUTHZ_ENABLED}
export GOHAN_PUBLIC_AUTHN_JWKS_URL=${GOHAN_PUBLIC_AUTHN_JWKS_URL}
export GOHAN_PRIVATE_AUTHZ_URL=${GOHAN_PRIVATE_AUTHZ_URL}
export GOHAN_AUTHZ_REQHEADS=${GOHAN_API_AUTHZ_REQHEADS}
<br />

<b>Local Development :</b>

# DRS
export GOHAN_DRS_URL=${GOHAN_PRIVATE_DRS_URL}
export GOHAN_DRS_BASIC_AUTH_USERNAME=${GOHAN_DRS_BASIC_AUTH_USERNAME}
export GOHAN_DRS_BASIC_AUTH_PASSWORD=${GOHAN_DRS_BASIC_AUTH_PASSWORD}
&nbsp;This can be done multiple ways.

1. `Terminal` : From the project root, run
```
# load variables from local file
set -a
. ./.env
set +a
cd src/api
go run .
```

2. `IDE (preferably VSCode)`

- follow the recommended instructions listed at https://code.visualstudio.com/docs/languages/go

- configure the `.vscode/launch.json` to inject the above mentioned variables as recommended by https://stackoverflow.com/questions/29971572/how-do-i-add-environment-variables-to-launch-json-in-vscode

- click 'Run & Debug' > "Play"

<b>Local Release</b>

&nbsp;To build / test from source;

```
make build-api-local-binaries
```


&nbsp;The binary can then be found at *bin/api_${GOOS}_${GOARCH}* and executed locally with

```
# load variables from local file
set -a
. ./.env
set +a
# navigate to binary directory
cd bin/
# execute binary
./api_${GOOS}_${GOARCH}
```



<b>Endpoints :</b>

Expand Down Expand Up @@ -315,9 +422,9 @@ Generalized Response Body Structure
<br />
- http://localhost:5000/vcfs/get/by/sampleId?chromosome=2&id=NA12815&size=10000
- http://localhost:5000/variants/get/by/sampleId?chromosome=2&id=NA12815&size=10000
- http://localhost:5000/vcfs/get/by/sampleId?chromosome=2&id=NA12815&size=1000&lowerBound=1000&upperBound=100000
- http://localhost:5000/variants/get/by/sampleId?chromosome=2&id=NA12815&size=1000&lowerBound=1000&upperBound=100000
<br />
Expand Down Expand Up @@ -485,63 +592,8 @@ Response
`Status Code:` **204**
<br />
## Releases
### **API :**
Local Release:
&nbsp;From the project root, run
```
make build-api-local-binaries
```
&nbsp;The binary can then be found at *bin/api_${GOOS}_${GOARCH}* and executed locally with
```
export GOHAN_API_INTERNAL_PORT=${GOHAN_API_INTERNAL_PORT}
export GOHAN_API_VCF_PATH=${GOHAN_API_VCF_PATH}

# Elasticsearch
export GOHAN_ES_URL=${GOHAN_PRIVATE_ES_URL}
export GOHAN_ES_USERNAME=${GOHAN_ES_USERNAME}
export GOHAN_ES_PASSWORD=${GOHAN_ES_PASSWORD}

# AuthX
export GOHAN_AUTHZ_ENABLED=${GOHAN_API_AUTHZ_ENABLED}
export GOHAN_PUBLIC_AUTHN_JWKS_URL=${GOHAN_PUBLIC_AUTHN_JWKS_URL}
export GOHAN_PRIVATE_AUTHZ_URL=${GOHAN_PRIVATE_AUTHZ_URL}
export GOHAN_AUTHZ_REQHEADS=${GOHAN_API_AUTHZ_REQHEADS}

# DRS
export GOHAN_DRS_URL=${GOHAN_PRIVATE_DRS_URL}
export GOHAN_DRS_BASIC_AUTH_USERNAME=${GOHAN_DRS_BASIC_AUTH_USERNAME}
export GOHAN_DRS_BASIC_AUTH_PASSWORD=${GOHAN_DRS_BASIC_AUTH_PASSWORD}

cd bin/

./api_${GOOS}_${GOARCH}
```
<br />
Containerized Alpine Release:
&nbsp;When ready, build the `docker image` and spawn the `container` with an independent binary build by running
```
make build-api
make run-api
```
&nbsp;and the `docker-compose.yaml` file will handle the configuration.
<br />
<br />
Expand Down Expand Up @@ -570,3 +622,4 @@ Once `elasticsearch`, `drs`, the `api`, and the `gateway` are up, run
```
make test-api-dev
```
Loading

0 comments on commit 037ffa6

Please sign in to comment.