Merge pull request #290 from ethpandaops/feat/local-clickhouse

feat(docker): add clickhouse profile
ethpandaops · Mar 25, 2024 · 12a01f5 · 12a01f5
2 parents ac8765f + 6cae855
commit 12a01f5
Show file tree

Hide file tree

Showing 2 changed files with 103 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -45,31 +45,37 @@ Xatu can run in multiple modes. Each mode can be run independently. The followin
 Follow the links for more information on each mode.
 
 - [**Server**](./docs/server.md) - Centralized server collecting events from various clients and can output them to various sinks.
-- [**Sentry**](./docs/sentry.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). *You must run your own consensus client* and this projects sentry will connect to it via the consensus client's http server.
+- [**Sentry**](./docs/sentry.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). _You must run your own consensus client_ and this projects sentry will connect to it via the consensus client's http server.
 - [**Discovery**](./docs/discovery.md) - Client that uses the [Node Discovery Protocol v5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) and [Node Discovery Protocol v4](https://github.com/ethereum/devp2p/blob/master/discv4.md) to discovery nodes on the network. Also attempts to connect to execution layer nodes and collect meta data from them.
 - [**Mimicry**](./docs/mimicry.md) - Client that collects data from the execution layer P2P network.
-- [**Cannon**](./docs/cannon.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects canonical finalized data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). *You must run your own consensus client* and this projects cannon client will connect to it via the consensus client's http server.
+- [**Cannon**](./docs/cannon.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects canonical finalized data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). _You must run your own consensus client_ and this projects cannon client will connect to it via the consensus client's http server.
 - [**Sage**](./docs/sage.md) - Client that connects to an [Armiarma](https://github.com/migalabs/armiarma) instance (which itself connects to an Ethereum Beacon Chain P2P network) and creates events from the events Armiarma emits.
 
 ## Getting Started
 
 ### Download a release
+
 Download the latest release from the [Releases page](https://github.com/ethpandaops/xatu/releases). Extract and run with:
+
 ```
 ./xatu <server|sentry|discovery|mimicry> --config your-config.yaml
 ```
 
 ### Docker
+
 Available as a docker image at [ethpandaops/xatu](https://hub.docker.com/r/ethpandaops/xatu/tags)
 
 #### Images
+
 - `latest` - distroless, multiarch
 - `latest-debian` - debian, multiarch
 - `$version` - distroless, multiarch, pinned to a release (i.e. `0.4.0`)
 - `$version-debian` - debian, multiarch, pinned to a release (i.e. `0.4.0-debian`)
 
 ### Kubernetes via Helm
+
 [Read more](https://github.com/ethpandaops/ethereum-helm-charts/tree/master/charts/xatu)
+
 ```
 helm repo add ethereum-helm-charts https://ethpandaops.github.io/ethereum-helm-charts
 
@@ -87,16 +93,19 @@ This will setup a pipeline to import events from Xatu server into a clickhouse i
 There is also a grafana instance running with dashboards that can be used to visualize the data.
 
 Exposed ports:
+
 - `8080` - Xatu server
 - `9000` - Clickhouse native port
 - `8123` - Clickhouse http port
 - `3000` - Grafana
 
 Links:
+
 - [Clickhouse playground](http://localhost:8123/play)
 - [Grafana](http://localhost:3000)
 
 Example sentry config to connect to the server:
+
 ```yaml
 logging: "info"
 metricsAddr: ":9095"
@@ -134,17 +143,24 @@ outputs:
  connections: 3
 ```
 
+### Local clickhouse
+
+You can start up the clickhouse cluster only with migrations automatically applied. You might want to do this to play with out [Xatu data](https://github.com/ethpandaops/xatu-data) locally.
+
+```bash
+docker compose --profile clickhouse up --detach
+```
+
 ## Contributing
 
-Contributions are greatly appreciated! Pull requests will be reviewed and merged promptly if you're interested in improving Xatu! 
+Contributions are greatly appreciated! Pull requests will be reviewed and merged promptly if you're interested in improving Xatu!
 
 1. Fork the project
 2. Create your feature branch:
-  - `git checkout -b feat/new-output`
+ - `git checkout -b feat/new-output`
 3. Commit your changes:
- - `git commit -m 'feat(sentry): new output`
-4. Push to the branch:
- -`git push origin feat/new-output`
+ - `git commit -m 'feat(sentry): new output`
+4. Push to the branch: -`git push origin feat/new-output`
 5. Open a pull request
 
 ## Contact

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,10 @@
-version: '3.7'
+version: "3.7"
 
 services:
  clickhouse-01:
+ profiles:
+ - clickhouse
+ - ""
  image: "clickhouse/clickhouse-server:${CHVER:-latest}"
  container_name: clickhouse-01
  hostname: clickhouse-01
@@ -24,12 +27,19 @@ services:
  clickhouse-keeper-03:
  condition: service_healthy
  healthcheck:
- test: ["CMD-SHELL", "set -x; wget --spider --quiet http://clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1"]
+ test:
+ [
+ "CMD-SHELL",
+ "set -x; wget --spider --quiet http://clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1",
+ ]
  interval: 5s
  timeout: 10s
  retries: 15
  start_period: 15s
  clickhouse-02:
+ profiles:
+ - clickhouse
+ - ""
  image: "clickhouse/clickhouse-server:${CHVER:-latest}"
  container_name: clickhouse-02
  hostname: clickhouse-02
@@ -52,63 +62,78 @@ services:
  clickhouse-keeper-03:
  condition: service_healthy
  healthcheck:
- test: ["CMD-SHELL", "set -x; wget --spider --quiet http://clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1"]
+ test:
+ [
+ "CMD-SHELL",
+ "set -x; wget --spider --quiet http://clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1",
+ ]
  interval: 5s
  timeout: 10s
  retries: 15
  start_period: 15s
  clickhouse-keeper-01:
+ profiles:
+ - clickhouse
+ - ""
  image: "clickhouse/clickhouse-keeper:23.12-alpine"
  user: "101:101"
  container_name: clickhouse-keeper-01
  hostname: clickhouse-keeper-01
  networks:
  - xatu-net
  volumes:
- - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-01/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml
+  - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-01/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml
  ports:
-  - "127.0.0.1:9181:9181"
+ - "127.0.0.1:9181:9181"
  healthcheck:
  test: ["CMD-SHELL", "echo ruok | nc localhost 9181 | grep -q 'imok'"]
  interval: 30s
  timeout: 10s
  retries: 3
  start_period: 5s
  clickhouse-keeper-02:
+ profiles:
+ - clickhouse
+ - ""
  image: "clickhouse/clickhouse-keeper:23.12-alpine"
  user: "101:101"
  container_name: clickhouse-keeper-02
  hostname: clickhouse-keeper-02
  networks:
  - xatu-net
  volumes:
- - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-02/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml
+  - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-02/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml
  ports:
-  - "127.0.0.1:9182:9181"
+ - "127.0.0.1:9182:9181"
  healthcheck:
  test: ["CMD-SHELL", "echo ruok | nc localhost 9181 | grep -q 'imok'"]
  interval: 30s
  timeout: 10s
  retries: 3
  start_period: 5s
  clickhouse-keeper-03:
+ profiles:
+ - clickhouse
+ - ""
  image: "clickhouse/clickhouse-keeper:23.12-alpine"
  user: "101:101"
  container_name: clickhouse-keeper-03
  hostname: clickhouse-keeper-03
  networks:
  - xatu-net
  volumes:
- - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-03/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml
+  - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-03/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml
  ports:
-  - "127.0.0.1:9183:9181"
+ - "127.0.0.1:9183:9181"
  healthcheck:
  test: ["CMD-SHELL", "echo ruok | nc localhost 9181 | grep -q 'imok'"]
  interval: 30s
  timeout: 10s
  retries: 3
  start_period: 5s
  xatu-server:
+ profiles:
+ - ""
  image: ethpandaops/xatu:latest
  command: server --config /etc/xatu-server/config.yaml
  build:
@@ -123,6 +148,8 @@ services:
  - xatu-net
 
  postgres:
+ profiles:
+ - ""
  image: postgres:15
  volumes:
  - postgres-data:/var/lib/postgresql/data
@@ -140,6 +167,8 @@ services:
  - xatu-net
 
  grafana:
+ profiles:
+ - ""
  image: grafana/grafana:latest
  ports:
  - "3000:3000"
@@ -156,6 +185,8 @@ services:
  - ./deploy/local/docker-compose/grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml
  - ./deploy/local/docker-compose/grafana/dashboards:/var/lib/grafana/dashboards
  prometheus:
+ profiles:
+ - ""
  image: prom/prometheus:latest
  ports:
  - "9090:9090"
@@ -166,10 +197,12 @@ services:
  - ./deploy/local/docker-compose/prometheus.yaml:/etc/prometheus/prometheus.yml
 
  init-kafka:
+ profiles:
+ - ""
  image: confluentinc/cp-kafka:latest
  hostname: init-kafka
  container_name: init-kafka
- entrypoint: 
+ entrypoint:
  - bash
  - -c
  command:
@@ -224,6 +257,8 @@ services:
  - xatu-net
 
  kafka:
+ profiles:
+ - ""
  image: confluentinc/cp-kafka:latest
  hostname: kafka
  container_name: kafka
@@ -236,14 +271,19 @@ services:
  KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
  KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
  KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
- KAFKA_LOG4J_LOGGERS: 'kafka.controller=ERROR,kafka.producer.async.DefaultEventHandler=ERROR,state.change.logger=ERROR'
+ KAFKA_LOG4J_LOGGERS: "kafka.controller=ERROR,kafka.producer.async.DefaultEventHandler=ERROR,state.change.logger=ERROR"
  ports:
  - "29092:29092"
  - "9092:9092"
  networks:
  - xatu-net
  healthcheck:
- test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server=localhost:9092"]
+ test:
+ [
+ "CMD",
+ "kafka-broker-api-versions",
+ "--bootstrap-server=localhost:9092",
+ ]
  interval: 30s
  timeout: 10s
  retries: 5
@@ -252,10 +292,12 @@ services:
  zookeeper:
  condition: service_healthy
  zookeeper:
+ profiles:
+ - ""
  image: zookeeper
  container_name: zookeeper
  environment:
- ZOO_LOG4J_PROP: 'ERROR,CONSOLE'
+ ZOO_LOG4J_PROP: "ERROR,CONSOLE"
  ports:
  - "2181:2181"
  networks:
@@ -268,6 +310,8 @@ services:
  start_period: 5s
 
  vector-http-kafka:
+ profiles:
+ - ""
  image: timberio/vector:0.34.1-alpine
  volumes:
  - ./deploy/local/docker-compose/vector-http-kafka.yaml:/etc/vector/vector.yaml
@@ -288,6 +332,8 @@ services:
  condition: service_healthy
 
  vector-kafka-clickhouse:
+ profiles:
+ - ""
  image: timberio/vector:0.34.1-alpine
  volumes:
  - ./deploy/local/docker-compose/vector-kafka-clickhouse.yaml:/etc/vector/vector.yaml
@@ -312,20 +358,39 @@ services:
  clickhouse-02:
  condition: service_healthy
  postgres-migrator:
+ profiles:
+ - ""
  image: migrate/migrate
  volumes:
  - ./migrations/postgres:/migrations
- command: ["-path", "/migrations", "-database", "postgres://user:password@postgres:5432/xatu?sslmode=disable", "up"]
+ command:
+ [
+ "-path",
+ "/migrations",
+ "-database",
+ "postgres://user:password@postgres:5432/xatu?sslmode=disable",
+ "up",
+ ]
  depends_on:
  postgres:
  condition: service_healthy
  networks:
  - xatu-net
  clickhouse-migrator:
+ profiles:
+ - clickhouse
+ - ""
  image: migrate/migrate
  volumes:
  - ./deploy/migrations/clickhouse:/migrations
- command: ["-path", "/migrations", "-database", "clickhouse://clickhouse-01:9000?username=default&database=default&x-multi-statement=true", "up"]
+ command:
+ [
+ "-path",
+ "/migrations",
+ "-database",
+ "clickhouse://clickhouse-01:9000?username=default&database=default&x-multi-statement=true",
+ "up",
+ ]
  depends_on:
  clickhouse-01:
  condition: service_healthy