From 6cae855d975335fc87c19e187ed64ab27baaaf6b Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:11:50 +1000 Subject: [PATCH] feat(docker): add clickhouse profile --- README.md | 30 +++++++++++---- docker-compose.yml | 95 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 103 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e74aec83..56aa6060 100644 --- a/README.md +++ b/README.md @@ -45,31 +45,37 @@ Xatu can run in multiple modes. Each mode can be run independently. The followin Follow the links for more information on each mode. - [**Server**](./docs/server.md) - Centralized server collecting events from various clients and can output them to various sinks. -- [**Sentry**](./docs/sentry.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). *You must run your own consensus client* and this projects sentry will connect to it via the consensus client's http server. +- [**Sentry**](./docs/sentry.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). _You must run your own consensus client_ and this projects sentry will connect to it via the consensus client's http server. - [**Discovery**](./docs/discovery.md) - Client that uses the [Node Discovery Protocol v5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) and [Node Discovery Protocol v4](https://github.com/ethereum/devp2p/blob/master/discv4.md) to discovery nodes on the network. Also attempts to connect to execution layer nodes and collect meta data from them. - [**Mimicry**](./docs/mimicry.md) - Client that collects data from the execution layer P2P network. -- [**Cannon**](./docs/cannon.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects canonical finalized data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). *You must run your own consensus client* and this projects cannon client will connect to it via the consensus client's http server. +- [**Cannon**](./docs/cannon.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects canonical finalized data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). _You must run your own consensus client_ and this projects cannon client will connect to it via the consensus client's http server. - [**Sage**](./docs/sage.md) - Client that connects to an [Armiarma](https://github.com/migalabs/armiarma) instance (which itself connects to an Ethereum Beacon Chain P2P network) and creates events from the events Armiarma emits. ## Getting Started ### Download a release + Download the latest release from the [Releases page](https://github.com/ethpandaops/xatu/releases). Extract and run with: + ``` ./xatu --config your-config.yaml ``` ### Docker + Available as a docker image at [ethpandaops/xatu](https://hub.docker.com/r/ethpandaops/xatu/tags) #### Images + - `latest` - distroless, multiarch - `latest-debian` - debian, multiarch - `$version` - distroless, multiarch, pinned to a release (i.e. `0.4.0`) - `$version-debian` - debian, multiarch, pinned to a release (i.e. `0.4.0-debian`) ### Kubernetes via Helm + [Read more](https://github.com/ethpandaops/ethereum-helm-charts/tree/master/charts/xatu) + ``` helm repo add ethereum-helm-charts https://ethpandaops.github.io/ethereum-helm-charts @@ -87,16 +93,19 @@ This will setup a pipeline to import events from Xatu server into a clickhouse i There is also a grafana instance running with dashboards that can be used to visualize the data. Exposed ports: + - `8080` - Xatu server - `9000` - Clickhouse native port - `8123` - Clickhouse http port - `3000` - Grafana Links: + - [Clickhouse playground](http://localhost:8123/play) - [Grafana](http://localhost:3000) Example sentry config to connect to the server: + ```yaml logging: "info" metricsAddr: ":9095" @@ -134,17 +143,24 @@ outputs: connections: 3 ``` +### Local clickhouse + +You can start up the clickhouse cluster only with migrations automatically applied. You might want to do this to play with out [Xatu data](https://github.com/ethpandaops/xatu-data) locally. + +```bash +docker compose --profile clickhouse up --detach +``` + ## Contributing -Contributions are greatly appreciated! Pull requests will be reviewed and merged promptly if you're interested in improving Xatu! +Contributions are greatly appreciated! Pull requests will be reviewed and merged promptly if you're interested in improving Xatu! 1. Fork the project 2. Create your feature branch: - - `git checkout -b feat/new-output` + - `git checkout -b feat/new-output` 3. Commit your changes: - - `git commit -m 'feat(sentry): new output` -4. Push to the branch: - -`git push origin feat/new-output` + - `git commit -m 'feat(sentry): new output` +4. Push to the branch: -`git push origin feat/new-output` 5. Open a pull request ## Contact diff --git a/docker-compose.yml b/docker-compose.yml index e8db4fb0..c1f7003f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,10 @@ -version: '3.7' +version: "3.7" services: clickhouse-01: + profiles: + - clickhouse + - "" image: "clickhouse/clickhouse-server:${CHVER:-latest}" container_name: clickhouse-01 hostname: clickhouse-01 @@ -24,12 +27,19 @@ services: clickhouse-keeper-03: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "set -x; wget --spider --quiet http://clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1"] + test: + [ + "CMD-SHELL", + "set -x; wget --spider --quiet http://clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1", + ] interval: 5s timeout: 10s retries: 15 start_period: 15s clickhouse-02: + profiles: + - clickhouse + - "" image: "clickhouse/clickhouse-server:${CHVER:-latest}" container_name: clickhouse-02 hostname: clickhouse-02 @@ -52,12 +62,19 @@ services: clickhouse-keeper-03: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "set -x; wget --spider --quiet http://clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1"] + test: + [ + "CMD-SHELL", + "set -x; wget --spider --quiet http://clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1", + ] interval: 5s timeout: 10s retries: 15 start_period: 15s clickhouse-keeper-01: + profiles: + - clickhouse + - "" image: "clickhouse/clickhouse-keeper:23.12-alpine" user: "101:101" container_name: clickhouse-keeper-01 @@ -65,9 +82,9 @@ services: networks: - xatu-net volumes: - - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-01/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml + - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-01/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml ports: - - "127.0.0.1:9181:9181" + - "127.0.0.1:9181:9181" healthcheck: test: ["CMD-SHELL", "echo ruok | nc localhost 9181 | grep -q 'imok'"] interval: 30s @@ -75,6 +92,9 @@ services: retries: 3 start_period: 5s clickhouse-keeper-02: + profiles: + - clickhouse + - "" image: "clickhouse/clickhouse-keeper:23.12-alpine" user: "101:101" container_name: clickhouse-keeper-02 @@ -82,9 +102,9 @@ services: networks: - xatu-net volumes: - - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-02/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml + - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-02/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml ports: - - "127.0.0.1:9182:9181" + - "127.0.0.1:9182:9181" healthcheck: test: ["CMD-SHELL", "echo ruok | nc localhost 9181 | grep -q 'imok'"] interval: 30s @@ -92,6 +112,9 @@ services: retries: 3 start_period: 5s clickhouse-keeper-03: + profiles: + - clickhouse + - "" image: "clickhouse/clickhouse-keeper:23.12-alpine" user: "101:101" container_name: clickhouse-keeper-03 @@ -99,9 +122,9 @@ services: networks: - xatu-net volumes: - - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-03/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml + - ./deploy/local/docker-compose/clickhouse/clickhouse-keeper-03/etc/clickhouse-keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml ports: - - "127.0.0.1:9183:9181" + - "127.0.0.1:9183:9181" healthcheck: test: ["CMD-SHELL", "echo ruok | nc localhost 9181 | grep -q 'imok'"] interval: 30s @@ -109,6 +132,8 @@ services: retries: 3 start_period: 5s xatu-server: + profiles: + - "" image: ethpandaops/xatu:latest command: server --config /etc/xatu-server/config.yaml build: @@ -123,6 +148,8 @@ services: - xatu-net postgres: + profiles: + - "" image: postgres:15 volumes: - postgres-data:/var/lib/postgresql/data @@ -140,6 +167,8 @@ services: - xatu-net grafana: + profiles: + - "" image: grafana/grafana:latest ports: - "3000:3000" @@ -156,6 +185,8 @@ services: - ./deploy/local/docker-compose/grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml - ./deploy/local/docker-compose/grafana/dashboards:/var/lib/grafana/dashboards prometheus: + profiles: + - "" image: prom/prometheus:latest ports: - "9090:9090" @@ -166,10 +197,12 @@ services: - ./deploy/local/docker-compose/prometheus.yaml:/etc/prometheus/prometheus.yml init-kafka: + profiles: + - "" image: confluentinc/cp-kafka:latest hostname: init-kafka container_name: init-kafka - entrypoint: + entrypoint: - bash - -c command: @@ -224,6 +257,8 @@ services: - xatu-net kafka: + profiles: + - "" image: confluentinc/cp-kafka:latest hostname: kafka container_name: kafka @@ -236,14 +271,19 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_LOG4J_LOGGERS: 'kafka.controller=ERROR,kafka.producer.async.DefaultEventHandler=ERROR,state.change.logger=ERROR' + KAFKA_LOG4J_LOGGERS: "kafka.controller=ERROR,kafka.producer.async.DefaultEventHandler=ERROR,state.change.logger=ERROR" ports: - "29092:29092" - "9092:9092" networks: - xatu-net healthcheck: - test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server=localhost:9092"] + test: + [ + "CMD", + "kafka-broker-api-versions", + "--bootstrap-server=localhost:9092", + ] interval: 30s timeout: 10s retries: 5 @@ -252,10 +292,12 @@ services: zookeeper: condition: service_healthy zookeeper: + profiles: + - "" image: zookeeper container_name: zookeeper environment: - ZOO_LOG4J_PROP: 'ERROR,CONSOLE' + ZOO_LOG4J_PROP: "ERROR,CONSOLE" ports: - "2181:2181" networks: @@ -268,6 +310,8 @@ services: start_period: 5s vector-http-kafka: + profiles: + - "" image: timberio/vector:0.34.1-alpine volumes: - ./deploy/local/docker-compose/vector-http-kafka.yaml:/etc/vector/vector.yaml @@ -288,6 +332,8 @@ services: condition: service_healthy vector-kafka-clickhouse: + profiles: + - "" image: timberio/vector:0.34.1-alpine volumes: - ./deploy/local/docker-compose/vector-kafka-clickhouse.yaml:/etc/vector/vector.yaml @@ -312,20 +358,39 @@ services: clickhouse-02: condition: service_healthy postgres-migrator: + profiles: + - "" image: migrate/migrate volumes: - ./migrations/postgres:/migrations - command: ["-path", "/migrations", "-database", "postgres://user:password@postgres:5432/xatu?sslmode=disable", "up"] + command: + [ + "-path", + "/migrations", + "-database", + "postgres://user:password@postgres:5432/xatu?sslmode=disable", + "up", + ] depends_on: postgres: condition: service_healthy networks: - xatu-net clickhouse-migrator: + profiles: + - clickhouse + - "" image: migrate/migrate volumes: - ./deploy/migrations/clickhouse:/migrations - command: ["-path", "/migrations", "-database", "clickhouse://clickhouse-01:9000?username=default&database=default&x-multi-statement=true", "up"] + command: + [ + "-path", + "/migrations", + "-database", + "clickhouse://clickhouse-01:9000?username=default&database=default&x-multi-statement=true", + "up", + ] depends_on: clickhouse-01: condition: service_healthy