From 931696b0e131880418798e5f432f3a290de1edcf Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:40:30 +1000 Subject: [PATCH 1/3] feat(docker): add volumes and clean up container names --- .../etc/clickhouse-server/config.d/config.xml | 10 +- .../etc/clickhouse-server/config.d/config.xml | 10 +- .../grafana/datasources/datasources.yaml | 8 +- deploy/local/docker-compose/prometheus.yaml | 4 +- deploy/local/docker-compose/xatu-server.yaml | 4 +- docker-compose.yml | 158 +++++++++++------- 6 files changed, 113 insertions(+), 81 deletions(-) diff --git a/deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml b/deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml index e5a0623e..df4238cc 100644 --- a/deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml +++ b/deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml @@ -22,13 +22,13 @@ - clickhouse-01 + xatu-clickhouse-01 9000 - clickhouse-02 + xatu-clickhouse-02 9000 @@ -36,15 +36,15 @@ - zookeeper-01 + xatu-clickhouse-zookeeper-01 2181 - zookeeper-02 + xatu-clickhouse-zookeeper-02 2181 - zookeeper-03 + xatu-clickhouse-zookeeper-03 2181 diff --git a/deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml b/deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml index f70ca5d6..26d6c71d 100644 --- a/deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml +++ b/deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml @@ -22,13 +22,13 @@ - clickhouse-01 + xatu-clickhouse-01 9000 - clickhouse-02 + xatu-clickhouse-02 9000 @@ -36,15 +36,15 @@ - zookeeper-01 + xatu-clickhouse-zookeeper-01 2181 - zookeeper-02 + xatu-clickhouse-zookeeper-02 2181 - zookeeper-03 + xatu-clickhouse-zookeeper-03 2181 diff --git a/deploy/local/docker-compose/grafana/datasources/datasources.yaml b/deploy/local/docker-compose/grafana/datasources/datasources.yaml index a74bb675..91ddd016 100644 --- a/deploy/local/docker-compose/grafana/datasources/datasources.yaml +++ b/deploy/local/docker-compose/grafana/datasources/datasources.yaml @@ -4,25 +4,25 @@ datasources: - name: Prometheus type: prometheus access: proxy - url: http://prometheus:9090 + url: http://xatu-prometheus:9090 - name: ClickHouse-official type: grafana-clickhouse-datasource jsonData: defaultDatabase: default port: 9000 - server: clickhouse-01 + server: xatu-clickhouse-01 tlsSkipVerify: true - name: ClickHouse-vert type: vertamedia-clickhouse-datasource access: proxy - url: http://clickhouse-01:8123 + url: http://xatu-clickhouse-01:8123 - name: postgres type: postgres access: proxy - url: postgres:5432 + url: xatu-postgres:5432 user: user secureJsonData: password: password diff --git a/deploy/local/docker-compose/prometheus.yaml b/deploy/local/docker-compose/prometheus.yaml index 633a39f6..39f3ca95 100644 --- a/deploy/local/docker-compose/prometheus.yaml +++ b/deploy/local/docker-compose/prometheus.yaml @@ -4,7 +4,7 @@ scrape_configs: - targets: ['xatu-server:9090'] - job_name: 'vector-kafka-clickhouse' static_configs: - - targets: ['vector-kafka-clickhouse:9598'] + - targets: ['xatu-vector-kafka-clickhouse:9598'] - job_name: 'vector-http-kafka' static_configs: - - targets: ['vector-http-kafka:9598'] \ No newline at end of file + - targets: ['xatu-vector-http-kafka:9598'] \ No newline at end of file diff --git a/deploy/local/docker-compose/xatu-server.yaml b/deploy/local/docker-compose/xatu-server.yaml index c596993b..f7559c37 100644 --- a/deploy/local/docker-compose/xatu-server.yaml +++ b/deploy/local/docker-compose/xatu-server.yaml @@ -16,7 +16,7 @@ ntpServer: time.google.com persistence: enabled: true driverName: postgres - connectionString: postgres://user:password@postgres:5432/xatu?sslmode=disable + connectionString: postgres://user:password@xatu-postgres:5432/xatu?sslmode=disable maxIdleConns: 2 maxOpenConns: 5 @@ -40,7 +40,7 @@ services: - name: general type: http config: - address: http://vector-http-kafka:9005 + address: http://xatu-vector-http-kafka:9005 maxQueueSize: 102400 batchTimeout: 1s exportTimeout: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 6c504e5e..e8e4c2b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,16 @@ services: - clickhouse-01: + xatu-clickhouse-01: profiles: - clickhouse - "" image: "clickhouse/clickhouse-server:${CHVER:-latest}" - container_name: clickhouse-01 - hostname: clickhouse-01 + container_name: xatu-clickhouse-01 + hostname: xatu-clickhouse-01 user: 101:101 networks: - xatu-net volumes: - # - clickhouse-01-data:/var/lib/clickhouse/ + - clickhouse-01-data:/var/lib/clickhouse/ - ./deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml - ./deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml - ./deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d @@ -18,34 +18,34 @@ services: - "127.0.0.1:8123:8123" - "127.0.0.1:9000:9000" depends_on: - zookeeper-01: + xatu-clickhouse-zookeeper-01: condition: service_healthy - zookeeper-02: + xatu-clickhouse-zookeeper-02: condition: service_healthy - zookeeper-03: + xatu-clickhouse-zookeeper-03: condition: service_healthy healthcheck: test: [ "CMD-SHELL", - "set -x; wget --spider --quiet http://clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1", + "set -x; wget --spider --quiet http://xatu-clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1", ] interval: 5s timeout: 10s retries: 15 start_period: 15s - clickhouse-02: + xatu-clickhouse-02: profiles: - clickhouse - "" image: "clickhouse/clickhouse-server:${CHVER:-latest}" - container_name: clickhouse-02 - hostname: clickhouse-02 + container_name: xatu-clickhouse-02 + hostname: xatu-clickhouse-02 user: 101:101 networks: - xatu-net volumes: - # - clickhouse-02-data:/var/lib/clickhouse/ + - clickhouse-02-data:/var/lib/clickhouse/ - ./deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml - ./deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml - ./deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d @@ -53,73 +53,82 @@ services: - "127.0.0.1:8124:8123" - "127.0.0.1:9001:9000" depends_on: - zookeeper-01: + xatu-clickhouse-zookeeper-01: condition: service_healthy - zookeeper-02: + xatu-clickhouse-zookeeper-02: condition: service_healthy - zookeeper-03: + xatu-clickhouse-zookeeper-03: condition: service_healthy healthcheck: test: [ "CMD-SHELL", - "set -x; wget --spider --quiet http://clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1", + "set -x; wget --spider --quiet http://xatu-clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1", ] interval: 5s timeout: 10s retries: 15 start_period: 15s - zookeeper-01: + xatu-clickhouse-zookeeper-01: profiles: - clickhouse - "" image: zookeeper:3.9 - container_name: zookeeper-01 - hostname: zookeeper-01 + container_name: xatu-clickhouse-zookeeper-01 + hostname: xatu-clickhouse-zookeeper-01 networks: - xatu-net + volumes: + - clickhouse-zookeeper-01-data:/data + - clickhouse-zookeeper-01-datalog:/datalog environment: ZOO_MY_ID: 1 ZOO_4LW_COMMANDS_WHITELIST: "*" - ZOO_SERVERS: "server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181" + ZOO_SERVERS: "server.1=xatu-clickhouse-zookeeper-01:2888:3888;2181 server.2=xatu-clickhouse-zookeeper-02:2888:3888;2181 server.3=xatu-clickhouse-zookeeper-03:2888:3888;2181" healthcheck: test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep -q 'imok'"] interval: 30s timeout: 10s retries: 3 start_period: 5s - zookeeper-02: + xatu-clickhouse-zookeeper-02: profiles: - clickhouse - "" image: zookeeper:3.9 - container_name: zookeeper-02 - hostname: zookeeper-02 + container_name: xatu-clickhouse-zookeeper-02 + hostname: xatu-clickhouse-zookeeper-02 networks: - xatu-net + volumes: + - clickhouse-zookeeper-02-data:/data + - clickhouse-zookeeper-02-datalog:/datalog environment: ZOO_MY_ID: 2 ZOO_4LW_COMMANDS_WHITELIST: "*" - ZOO_SERVERS: "server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181" + ZOO_SERVERS: "server.1=xatu-clickhouse-zookeeper-01:2888:3888;2181 server.2=xatu-clickhouse-zookeeper-02:2888:3888;2181 server.3=xatu-clickhouse-zookeeper-03:2888:3888;2181" healthcheck: test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep -q 'imok'"] interval: 30s timeout: 10s retries: 3 start_period: 5s - zookeeper-03: + xatu-clickhouse-zookeeper-03: profiles: - clickhouse - "" image: zookeeper:3.9 - container_name: zookeeper-03 - hostname: zookeeper-03 + container_name: xatu-clickhouse-zookeeper-03 + hostname: xatu-clickhouse-zookeeper-03 networks: - xatu-net + volumes: + - clickhouse-zookeeper-03-data:/data + - clickhouse-zookeeper-03-datalog:/datalog environment: ZOO_MY_ID: 3 ZOO_4LW_COMMANDS_WHITELIST: "*" - ZOO_SERVERS: "server.1=zookeeper-01:2888:3888;2181 server.2=zookeeper-02:2888:3888;2181 server.3=zookeeper-03:2888:3888;2181" + ZOO_SERVERS: "server.1=xatu-clickhouse-zookeeper-01:2888:3888;2181 server.2=xatu-clickhouse-zookeeper-02:2888:3888;2181 server.3=xatu-clickhouse-zookeeper-03:2888:3888;2181" healthcheck: test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep -q 'imok'"] interval: 30s @@ -141,7 +150,7 @@ services: networks: - xatu-net - postgres: + xatu-postgres: profiles: - "" image: postgres:15 @@ -160,7 +169,7 @@ services: networks: - xatu-net - grafana: + xatu-grafana: profiles: - "" image: grafana/grafana:latest @@ -178,7 +187,7 @@ services: - ./deploy/local/docker-compose/grafana/datasources:/etc/grafana/provisioning/datasources - ./deploy/local/docker-compose/grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml - ./deploy/local/docker-compose/grafana/dashboards:/var/lib/grafana/dashboards - prometheus: + xatu-prometheus: profiles: - "" image: prom/prometheus:latest @@ -190,12 +199,12 @@ services: - prometheus-data:/prometheus - ./deploy/local/docker-compose/prometheus.yaml:/etc/prometheus/prometheus.yml - init-kafka: + xatu-init-kafka: profiles: - "" image: confluentinc/cp-kafka:latest - hostname: init-kafka - container_name: init-kafka + hostname: xatu-init-kafka + container_name: xatu-init-kafka entrypoint: - bash - -c @@ -251,27 +260,27 @@ services: ) for topic in "$${topics[@]}"; do echo "Creating topic: $$topic"; - kafka-topics --create --if-not-exists --bootstrap-server kafka:29092 --partitions 1 --replication-factor 1 --config cleanup.policy=compact --config retention.ms=300000 --topic "$$topic" + kafka-topics --create --if-not-exists --bootstrap-server xatu-kafka:29092 --partitions 1 --replication-factor 1 --config cleanup.policy=compact --config retention.ms=300000 --topic "$$topic" done sleep 3; depends_on: - kafka: + xatu-kafka: condition: service_healthy networks: - xatu-net - kafka: + xatu-kafka: profiles: - "" image: confluentinc/cp-kafka:latest - hostname: kafka - container_name: kafka + hostname: xatu-kafka + container_name: xatu-kafka environment: KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECT: xatu-kafka-zookeeper:2181 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://xatu-kafka:29092,PLAINTEXT_HOST://localhost:9092 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 @@ -282,6 +291,8 @@ services: - "9092:9092" networks: - xatu-net + volumes: + - kafka-data:/var/lib/kafka/data healthcheck: test: [ @@ -294,19 +305,22 @@ services: retries: 5 start_period: 5s depends_on: - zookeeper: + xatu-kafka-zookeeper: condition: service_healthy - zookeeper: + xatu-kafka-zookeeper: profiles: - "" image: zookeeper - container_name: zookeeper + container_name: xatu-kafka-zookeeper environment: ZOO_LOG4J_PROP: "ERROR,CONSOLE" ports: - "2181:2181" networks: - xatu-net + volumes: + - kafka-zookeeper-data:/data + - kafka-zookeeper-datalog:/datalog healthcheck: test: ["CMD", "./bin/zkServer.sh", "status"] interval: 30s @@ -314,14 +328,14 @@ services: retries: 5 start_period: 5s - vector-http-kafka: + xatu-vector-http-kafka: profiles: - "" image: timberio/vector:0.34.1-alpine volumes: - ./deploy/local/docker-compose/vector-http-kafka.yaml:/etc/vector/vector.yaml environment: - KAFKA_BROKERS: "kafka:29092" + KAFKA_BROKERS: "xatu-kafka:29092" ports: - "9005:9005" networks: @@ -333,12 +347,12 @@ services: retries: 5 start_period: 5s depends_on: - kafka: + xatu-kafka: condition: service_healthy - init-kafka: + xatu-init-kafka: condition: service_completed_successfully - vector-kafka-clickhouse: + xatu-vector-kafka-clickhouse: profiles: - "" image: timberio/vector:0.37.1-alpine @@ -349,10 +363,10 @@ services: ports: - "8686:8686" environment: - CLICKHOUSE_ENDPOINT: "clickhouse-01:8123" + CLICKHOUSE_ENDPOINT: "xatu-clickhouse-01:8123" CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: "" - KAFKA_BROKERS: "kafka:29092" + KAFKA_BROKERS: "xatu-kafka:29092" healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8686/health"] interval: 30s @@ -360,17 +374,17 @@ services: retries: 5 start_period: 5s depends_on: - kafka: + xatu-kafka: condition: service_healthy - clickhouse-01: + xatu-clickhouse-01: condition: service_healthy - clickhouse-02: + xatu-clickhouse-02: condition: service_healthy - clickhouse-migrator: + xatu-clickhouse-migrator: condition: service_completed_successfully - init-kafka: + xatu-init-kafka: condition: service_completed_successfully - postgres-migrator: + xatu-postgres-migrator: profiles: - "" image: migrate/migrate @@ -381,15 +395,15 @@ services: "-path", "/migrations", "-database", - "postgres://user:password@postgres:5432/xatu?sslmode=disable", + "postgres://user:password@xatu-postgres:5432/xatu?sslmode=disable", "up", ] depends_on: - postgres: + xatu-postgres: condition: service_healthy networks: - xatu-net - clickhouse-migrator: + xatu-clickhouse-migrator: profiles: - clickhouse - "" @@ -401,13 +415,13 @@ services: "-path", "/migrations", "-database", - "clickhouse://clickhouse-01:9000?username=default&database=default&x-multi-statement=true", + "clickhouse://xatu-clickhouse-01:9000?username=default&database=default&x-multi-statement=true", "up", ] depends_on: - clickhouse-01: + xatu-clickhouse-01: condition: service_healthy - clickhouse-02: + xatu-clickhouse-02: condition: service_healthy networks: - xatu-net @@ -423,5 +437,23 @@ volumes: driver: local clickhouse-02-data: driver: local + clickhouse-zookeeper-01-data: + driver: local + clickhouse-zookeeper-02-data: + driver: local + clickhouse-zookeeper-03-data: + driver: local + clickhouse-zookeeper-01-datalog: + driver: local + clickhouse-zookeeper-02-datalog: + driver: local + clickhouse-zookeeper-03-datalog: + driver: local + kafka-data: + driver: local + kafka-zookeeper-data: + driver: local + kafka-zookeeper-datalog: + driver: local prometheus-data: driver: local From d0ebcab73baa82a9de05cae95eb163bf64b94680 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:47:42 +1000 Subject: [PATCH 2/3] feat(docker): add volumes and clean up container names --- README.md | 16 ++++++++++++++++ docker-compose.yml | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/README.md b/README.md index 56aa6060..be7affbf 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,22 @@ You can start up the clickhouse cluster only with migrations automatically appli docker compose --profile clickhouse up --detach ``` +### Building xatu changes + +When making changes to the xatu codebase, you can build the docker image with the following command: + +```bash +docker compose up -d --build +``` + +### Cleaning up docker volumes + +We use docker volumes to store data between restarts. If you would like to clean up the volumes, you can do so with the following command: + +```bash +docker compose down -v +``` + ## Contributing Contributions are greatly appreciated! Pull requests will be reviewed and merged promptly if you're interested in improving Xatu! diff --git a/docker-compose.yml b/docker-compose.yml index e8e4c2b6..055d1fc5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,6 +139,8 @@ services: profiles: - "" command: server --config /etc/xatu-server/config.yaml + container_name: xatu-server + hostname: xatu-server build: context: . dockerfile: Dockerfile @@ -154,6 +156,8 @@ services: profiles: - "" image: postgres:15 + container_name: xatu-postgres + hostname: xatu-postgres volumes: - postgres-data:/var/lib/postgresql/data environment: @@ -173,6 +177,7 @@ services: profiles: - "" image: grafana/grafana:latest + container_name: xatu-grafana ports: - "3000:3000" networks: @@ -332,6 +337,7 @@ services: profiles: - "" image: timberio/vector:0.34.1-alpine + container_name: xatu-vector-http-kafka volumes: - ./deploy/local/docker-compose/vector-http-kafka.yaml:/etc/vector/vector.yaml environment: @@ -356,6 +362,7 @@ services: profiles: - "" image: timberio/vector:0.37.1-alpine + container_name: xatu-vector-kafka-clickhouse volumes: - ./deploy/local/docker-compose/vector-kafka-clickhouse.yaml:/etc/vector/vector.yaml networks: @@ -388,6 +395,7 @@ services: profiles: - "" image: migrate/migrate + container_name: xatu-postgres-migrator volumes: - ./migrations/postgres:/migrations command: @@ -408,6 +416,7 @@ services: - clickhouse - "" image: migrate/migrate + container_name: xatu-clickhouse-migrator volumes: - ./deploy/migrations/clickhouse:/migrations command: From ba5b246101b9a297afe12493951e8e967b97f9b4 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Tue, 7 May 2024 20:34:09 +1000 Subject: [PATCH 3/3] fix docker compose --- docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 260c6f1e..eef56856 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -398,7 +398,7 @@ services: - "" image: timberio/vector:0.37.1-alpine volumes: - - ./deploy/local/docker-compose/xatu-vector-kafka-clickhouse-libp2p.yaml:/etc/vector/vector.yaml + - ./deploy/local/docker-compose/vector-kafka-clickhouse-libp2p.yaml:/etc/vector/vector.yaml networks: - xatu-net environment: @@ -413,15 +413,15 @@ services: retries: 5 start_period: 5s depends_on: - kafka: + xatu-kafka: condition: service_healthy - clickhouse-01: + xatu-clickhouse-01: condition: service_healthy - clickhouse-02: + xatu-clickhouse-02: condition: service_healthy - clickhouse-migrator: + xatu-clickhouse-migrator: condition: service_completed_successfully - init-kafka: + xatu-init-kafka: condition: service_completed_successfully xatu-postgres-migrator: profiles: