-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
58 lines (58 loc) · 1.82 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
version: "3.6"
volumes:
dfs:
name: "dfs"
driver: local
services:
spark-master:
image: apache/spark:${SPARK_TAG}
ports:
- ${SPARK_MASTER_HTTP_HOST_PORT}:8080
- ${SPARK_MASTER_HOST_PORT}:${SPARK_MASTER_CONTAINER_PORT}
command: /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master >> logs/spark-master.out
volumes:
- dfs:/var/data
networks:
static-network:
ipv4_address: ${SPARK_MASTER}
spark-worker-1:
image: apache/spark:${SPARK_TAG}
environment:
- SPARK_WORKER_DIR=/opt/spark/work-dir
- SPARK_WORKER_PORT=${SPARK_WORKER_1_CONTAINER_PORT}
- SPARK_WORKER_CORES=1
- SPARK_WORKER_MEMORY=2048m
command: /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://${SPARK_MASTER}:${SPARK_MASTER_CONTAINER_PORT}
ports:
- ${SPARK_WORKER_1_HOST_PORT}:${SPARK_WORKER_1_CONTAINER_PORT}
- ${SPARK_WORKER_1_WEB_CONTAINER_PORT}:8081
volumes:
- dfs:/var/data
depends_on:
- spark-master
networks:
static-network:
ipv4_address: ${SPARK_WORKER_1}
spark-worker-2:
image: apache/spark:${SPARK_TAG}
environment:
- SPARK_WORKER_DIR=/opt/spark/work-dir
- SPARK_WORKER_PORT=${SPARK_WORKER_2_CONTAINER_PORT}
- SPARK_WORKER_CORES=1
- SPARK_WORKER_MEMORY=2048m
command: /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://${SPARK_MASTER}:${SPARK_MASTER_CONTAINER_PORT}
ports:
- ${SPARK_WORKER_2_HOST_PORT}:${SPARK_WORKER_2_CONTAINER_PORT}
- ${SPARK_WORKER_2_WEB_CONTAINER_PORT}:8082
volumes:
- dfs:/var/data
depends_on:
- spark-master
networks:
static-network:
ipv4_address: ${SPARK_WORKER_2}
networks:
static-network:
ipam:
config:
- subnet: ${CONTAINERS_NETWORK_SUBNET}