-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
98 lines (92 loc) · 2.71 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
version: '3'
services:
### --- SPARK SERVICES ---- ###
spark-master:
image: bitnami/spark:3.5.0
container_name: spark-master
hostname: spark-master
environment:
- SPARK_MODE=master
- SPARK_DAEMON_MEMORY=16g
- SPARK_WORKER_INSTANCES=2
- SPARK_EXECUTOR_MEMORY=16g
- SPARK_WORKER_CORES=8
ports:
- "8080:8080" # Web UI için
- "7077:7077" # Master URL
volumes:
- ./raw-data/orders.json:/raw-data/orders.json
- ./raw-data/products.json:/raw-data/products.json
deploy:
resources:
limits:
cpus: "8" # Increase the number of CPUs
memory: "16G" # Increase the amount of memory
restart: on-failure
networks:
- spark-network
spark-worker-1:
image: bitnami/spark:3.5.0
container_name: spark-worker-1
hostname: spark-worker-1
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark-master:7077
- SPARK_WORKER_MEMORY=8g
- SPARK_EXECUTOR_MEMORY=8g
volumes:
- ./raw-data/orders.json:/raw-data/orders.json
- ./raw-data/products.json:/raw-data/products.json
depends_on:
- spark-master
deploy:
resources:
limits:
cpus: "8" # Increase the number of CPUs
memory: "16G" # Increase the amount of memory
restart: on-failure
networks:
- spark-network
spark-worker-2:
image: bitnami/spark:3.5.0
container_name: spark-worker-2
hostname: spark-worker-2
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark-master:7077
- SPARK_WORKER_MEMORY=8g
- SPARK_EXECUTOR_MEMORY=8g
volumes:
- ./raw-data/orders.json:/raw-data/orders.json
- ./raw-data/products.json:/raw-data/products.json
depends_on:
- spark-master
deploy:
resources:
limits:
cpus: "8" # Increase the number of CPUs
memory: "16G" # Increase the amount of memory
restart: on-failure
networks:
- spark-network
### --- SPARK SUBMITTER --- ###
spark-submit:
container_name: spark-submit
build:
context: ./spark-submitter
dockerfile: Dockerfile
volumes:
- ./spark-submitter/Job1-1.0-SNAPSHOT-jar-with-dependencies.jar:/opt/spark/app/Job1-1.0-SNAPSHOT-jar-with-dependencies.jar
- ./spark-submitter/Job2-1.0-SNAPSHOT-jar-with-dependencies.jar:/opt/spark/app/Job2-1.0-SNAPSHOT-jar-with-dependencies.jar
- ./raw-data/orders.json:/raw-data/orders.json
- ./raw-data/products.json:/raw-data/products.json
depends_on:
- spark-master
- spark-worker-1
- spark-worker-2
restart: on-failure
networks:
- spark-network
networks:
spark-network:
external: true