generated from NOAA-OWP/owp-open-source-project-template
-
Notifications
You must be signed in to change notification settings - Fork 1
/
compose-workers.yml
165 lines (165 loc) · 9.06 KB
/
compose-workers.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
version: '2.3'
networks:
wres_net:
driver: bridge
enable_ipv6: false
ipam:
driver: default
config:
- subnet: 172.19.254.64/26
gateway: 172.19.254.65
volumes:
home:
driver_opts:
type: "nfs"
o: "ro,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,local_lock=none,addr=${NFS_HOME_DIR_IP_ADDRESS}"
device: "${NFS_HOME_DIR_DEVICE}"
services:
nginx:
image: "${DOCKER_REGISTRY}/wres/nginx"
container_name: nginx_proxy
restart: always
volumes:
- ${NGINX_SERVER_CERT}:/etc/nginx/cert/wres_server_cert.pem
- ${NGINX_SERVER_KEY}:/etc/nginx/cert/wres_server_key.pem
depends_on:
- cadvisor
ports:
- 9999:9999
networks:
wres_net:
cadvisor:
image: gcr.io/cadvisor/cadvisor:latest
container_name: cadvisor
restart: always
ports:
- 8080:8080
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /cgroup:/cgroup:ro
- /dev/disk:/dev/disk/:ro
command:
- --disable_metrics=network
- --docker_only=true
depends_on:
- worker
networks:
wres_net:
worker:
image: "${DOCKER_REGISTRY}/wres/wres-worker:20241210-76603a7"
restart: always
depends_on:
eventsbroker:
condition: service_started
volumes:
# To write user output data:
- /mnt/wres_share/evaluations:/mnt/wres_share/evaluations
# To read user input data using read-only nfs mount above:
- home:/home:ro
# To read user input data written by tasker:
- /mnt/wres_share/input_data:/mnt/wres_share/input_data:ro
# To read test input data:
- /mnt/wres_share/systests:/mnt/wres_share/systests:ro
# For certificates, keys that correspond to them, and .pgpass:
- /mnt/wres_keys:/wres_secrets:ro
# To write heap dumps (worker-shim process):
- /mnt/wres_share/heap_dumps/worker-shim:/mnt/wres_share/heap_dumps/worker-shim
# To write heap dumps (core WRES process):
- /mnt/wres_share/heap_dumps/wres:/mnt/wres_share/heap_dumps/wres
# For logs (inside container, avoid writing to read-only /home)
- /container_home
# Writing all log outputs:
- /mnt/wres_share/logs/worker/:/mnt/wres_share/logs/worker/
environment:
- JAVA_OPTS=-Dwres.broker=${WRES_BROKER_HOST} -Dcom.redhat.fips=false -Dwres.trustStore=${WRES_TRUST_STORE} -Dwres.trustStorePassword=${WRES_TRUST_STORE_PASSWORD} -Dwres.workerPathToClientP12Bundle=${WRES_WORKER_CLIENT_P12} -Dwres.workerPathToClientP12Password=${WRES_WORKER_CLIENT_P12_PASSWORD} -Djava.io.tmpdir=/mnt/wres_share/evaluations -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/worker-shim -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/worker-shim/java_pid%p.hprof /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/worker-shim/java_pid%p_$$CON_HOSTNAME.hprof'
- PGPASSFILE=/wres_secrets/.pgpass
# Use caller-specified db hostname from env var WRES_DB_FQDN.
# Do not auto-liquibase-migrate on each evaluation. This requires an
# administrator to run the migration(s) during or after deployment.
# Write heap dumps to the root of the volume above.
# Override the broker address and port in the binding URL of the event.properties.
- INNER_JAVA_OPTS=-Xmx8192m -Xms8192m -Dwres.maxPoolSize=13 -Dlogback.configurationFile=/opt/inner_logback.xml -Dwres.startBroker=false -Dwres.enableServerCache=false -Dwres.eventsBrokerAddress=eventsbroker -Dwres.eventsBrokerPort=5673 -Dwres.externalGraphics=true -Dwres.externalNumerics=true -Dwres.featureBatchSize=1 -Dwres.useDatabase=true -Dcom.redhat.fips=false -Djava.io.tmpdir=/mnt/wres_share/evaluations -Dwres.databaseHost=${WRES_DB_FQDN} -Dwres.certificateFileToTrust=${WRES_DB_CA_FILE} -Dwres.wrdsCertificateFileToTrust=${WRDS_CA_FILE} -Dwres.databaseName=wres8 -Dwres.username=wres_user8 -Dwres.attemptToMigrate=false -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/wres -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/wres/java_pid%p.hprof /mnt/wres_share/heap_dumps/wres/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/wres/java_pid%p_$$CON_HOSTNAME.hprof' -XX:StartFlightRecording=settings=wres_jfr,maxsize=1280m,dumponexit=true,filename=/mnt/wres_share/heap_dumps/wres/$$CON_HOSTNAME -XX:FlightRecorderOptions=repository=/mnt/wres_share/heap_dumps/wres/$$CON_HOSTNAME/ -XX:+CrashOnOutOfMemoryError -Dtruststore=${DSTORE_TRUST_STORE} -Dtruststorepassword=${DSTORE_TRUST_STORE_PASS}
- LANG=C.UTF-8
# Shim JVM should have 64m max heap, core JVM should have 2560m max heap
mem_limit: 12288m
cap_drop:
- ALL
read_only: true
healthcheck:
start_period: 1m
networks:
wres_net:
eventsbroker:
expose:
- 5673
ports:
- 15673:15673
image: "${DOCKER_REGISTRY}/wres/wres-eventsbroker:20241210-b9a6ab6"
restart: always
volumes:
# For heap-dump files
- /mnt/wres_share/heap_dumps/eventsbroker:/mnt/wres_share/heap_dumps/eventsbroker
# For certificates and keys that correspond to them
- /mnt/wres_keys:/wres_secrets:ro
environment:
- BROKER_WORK=/container_home
- BROKER_KEYSTORE_PATH=${EVENTSBROKER_KEYSTORE_PATH}
- BROKER_KEYSTORE_PASSWORD=${EVENTSBROKER_KEYSTORE_PASSWORD}
- BROKER_TRUSTSTORE_PATH=${EVENTSBROKER_TRUSTSTORE_PATH}
- BROKER_TRUSTSTORE_PASSWORD=${EVENTSBROKER_TRUSTSTORE_PASSWORD}
- JAVA_ARGS=-XX:+PrintClassHistogram -XX:+UseG1GC -XX:+UseStringDeduplication -Xms2048m -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/eventsbroker -Dcom.redhat.fips=false
- LANG=C.UTF-8
# Broker heap is 2048m in JAVA_ARGS
mem_limit: 2560m
cap_drop:
- ALL
read_only: false
networks:
wres_net:
graphics:
depends_on:
eventsbroker:
condition: service_started
image: "${DOCKER_REGISTRY}/wres/wres-graphics:20241209-33eacdb"
restart: always
volumes:
# To write graphics outputs
- /mnt/wres_share/evaluations:/mnt/wres_share/evaluations
# To write heap dumps
- /mnt/wres_share/heap_dumps/graphics:/mnt/wres_share/heap_dumps/graphics
environment:
# Override the broker address and port in the binding URL of the event.properties
- JAVA_OPTS=-Dwres.startBroker=false -Dwres.eventsBrokerAddress=eventsbroker -Dwres.eventsBrokerPort=5673 -Dcom.redhat.fips=false -Djava.io.tmpdir=/mnt/wres_share/evaluations -Xms3640m -Xmx3640m -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/graphics -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/graphics/java_pid%p.hprof /mnt/wres_share/heap_dumps/graphics/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/graphics/java_pid%p_$$CON_HOSTNAME.hprof' -XX:StartFlightRecording=name=graphics,maxsize=256m,disk=true,maxage=24h,dumponexit=true,filename=/mnt/wres_share/heap_dumps/graphics/$$CON_HOSTNAME -XX:FlightRecorderOptions=repository=/mnt/wres_share/heap_dumps/graphics/$$CON_HOSTNAME
- LANG=C.UTF-8
# Graphics limit is 3640m in JAVA_OPTS
mem_limit: 4096m
cap_drop:
- ALL
read_only: true
networks:
wres_net:
writing:
depends_on:
eventsbroker:
condition: service_started
image: "${DOCKER_REGISTRY}/wres/wres-writing:20241209-33eacdb"
restart: always
volumes:
# To write numeric outputs
- /mnt/wres_share/evaluations:/mnt/wres_share/evaluations
# To write heap dumps
- /mnt/wres_share/heap_dumps/writing:/mnt/wres_share/heap_dumps/writing
environment:
# Override the broker address and port in the binding URL of the event.properties
- JAVA_OPTS=-Dwres.startBroker=false -Dwres.eventsBrokerAddress=eventsbroker -Dwres.eventsBrokerPort=5673 -Dcom.redhat.fips=false -Djava.io.tmpdir=/mnt/wres_share/evaluations -Xms3640m -Xmx3640m -XX:HeapDumpPath=/mnt/wres_share/heap_dumps/writing -XX:OnOutOfMemoryError='mv /mnt/wres_share/heap_dumps/writing/java_pid%p.hprof /mnt/wres_share/heap_dumps/writing/java_pid%p_$$CON_HOSTNAME.hprof; chmod 775 /mnt/wres_share/heap_dumps/writing/java_pid%p_$$CON_HOSTNAME.hprof' -XX:StartFlightRecording=name=writing,maxsize=256m,disk=true,maxage=24h,dumponexit=true,filename=/mnt/wres_share/heap_dumps/writing/$$CON_HOSTNAME -XX:FlightRecorderOptions=repository=/mnt/wres_share/heap_dumps/writing/$$CON_HOSTNAME
- LANG=C.UTF-8
# Writing limit is 3640m in JAVA_OPTS
mem_limit: 4096m
cap_drop:
- ALL
read_only: true
networks:
wres_net: