docker-compose.yml

services:
  xatu-clickhouse-01:
    profiles:
      - clickhouse
      - ""
    image: "clickhouse/clickhouse-server:${CHVER:-latest}"
    container_name: xatu-clickhouse-01
    hostname: xatu-clickhouse-01
    user: 101:101
    networks:
      - xatu-net
    volumes:
      - clickhouse-01-data:/var/lib/clickhouse/
      - ./deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
      - ./deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
      - ./deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
    ports:
      - "${CLICKHOUSE_01_HTTP_ADDRESS:-127.0.0.1}:${CLICKHOUSE_01_HTTP_PORT:-8123}:8123"
      - "${CLICKHOUSE_01_NATIVE_ADDRESS:-127.0.0.1}:${CLICKHOUSE_01_NATIVE_PORT:-9000}:9000"
    depends_on:
      xatu-clickhouse-zookeeper-01:
        condition: service_healthy
      xatu-clickhouse-zookeeper-02:
        condition: service_healthy
      xatu-clickhouse-zookeeper-03:
        condition: service_healthy
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "set -x; wget --spider --quiet http://xatu-clickhouse-01:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1",
        ]
      interval: 5s
      timeout: 10s
      retries: 15
      start_period: 15s
  xatu-clickhouse-02:
    profiles:
      - clickhouse
      - ""
    image: "clickhouse/clickhouse-server:${CHVER:-latest}"
    container_name: xatu-clickhouse-02
    hostname: xatu-clickhouse-02
    user: 101:101
    networks:
      - xatu-net
    volumes:
      - clickhouse-02-data:/var/lib/clickhouse/
      - ./deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
      - ./deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
      - ./deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
    ports:
      - "${CLICKHOUSE_02_HTTP_ADDRESS:-127.0.0.1}:${CLICKHOUSE_02_HTTP_PORT:-8124}:8123"
      - "${CLICKHOUSE_02_NATIVE_ADDRESS:-127.0.0.1}:${CLICKHOUSE_02_NATIVE_PORT:-9001}:9000"
    depends_on:
      xatu-clickhouse-zookeeper-01:
        condition: service_healthy
      xatu-clickhouse-zookeeper-02:
        condition: service_healthy
      xatu-clickhouse-zookeeper-03:
        condition: service_healthy
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "set -x; wget --spider --quiet http://xatu-clickhouse-02:9000 || exit 1; wget --spider --quiet --header 'Host: localhost' --post-data 'query=SELECT 1' http://localhost:8123 || exit 1",
        ]
      interval: 5s
      timeout: 10s
      retries: 15
      start_period: 15s
  xatu-clickhouse-zookeeper-01:
    profiles:
      - clickhouse
      - ""
    image: zookeeper:3.9
    container_name: xatu-clickhouse-zookeeper-01
    hostname: xatu-clickhouse-zookeeper-01
    networks:
      - xatu-net
    volumes:
      - clickhouse-zookeeper-01-data:/data
      - clickhouse-zookeeper-01-datalog:/datalog
    environment:
      ZOO_MY_ID: 1
      ZOO_4LW_COMMANDS_WHITELIST: "*"
      ZOO_SERVERS: "server.1=xatu-clickhouse-zookeeper-01:2888:3888;2181 server.2=xatu-clickhouse-zookeeper-02:2888:3888;2181 server.3=xatu-clickhouse-zookeeper-03:2888:3888;2181"
      ZOO_AUTOPURGE_PURGEINTERVAL: 1
      ZOO_AUTOPURGE_SNAPRETAINCOUNT: 3
    healthcheck:
      test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep -q 'imok'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 5s
  xatu-clickhouse-zookeeper-02:
    profiles:
      - clickhouse
      - ""
    image: zookeeper:3.9
    container_name: xatu-clickhouse-zookeeper-02
    hostname: xatu-clickhouse-zookeeper-02
    networks:
      - xatu-net
    volumes:
      - clickhouse-zookeeper-02-data:/data
      - clickhouse-zookeeper-02-datalog:/datalog
    environment:
      ZOO_MY_ID: 2
      ZOO_4LW_COMMANDS_WHITELIST: "*"
      ZOO_SERVERS: "server.1=xatu-clickhouse-zookeeper-01:2888:3888;2181 server.2=xatu-clickhouse-zookeeper-02:2888:3888;2181 server.3=xatu-clickhouse-zookeeper-03:2888:3888;2181"
      ZOO_AUTOPURGE_PURGEINTERVAL: 1
      ZOO_AUTOPURGE_SNAPRETAINCOUNT: 3
    healthcheck:
      test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep -q 'imok'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 5s
  xatu-clickhouse-zookeeper-03:
    profiles:
      - clickhouse
      - ""
    image: zookeeper:3.9
    container_name: xatu-clickhouse-zookeeper-03
    hostname: xatu-clickhouse-zookeeper-03
    networks:
      - xatu-net
    volumes:
      - clickhouse-zookeeper-03-data:/data
      - clickhouse-zookeeper-03-datalog:/datalog
    environment:
      ZOO_MY_ID: 3
      ZOO_4LW_COMMANDS_WHITELIST: "*"
      ZOO_SERVERS: "server.1=xatu-clickhouse-zookeeper-01:2888:3888;2181 server.2=xatu-clickhouse-zookeeper-02:2888:3888;2181 server.3=xatu-clickhouse-zookeeper-03:2888:3888;2181"
      ZOO_AUTOPURGE_PURGEINTERVAL: 1
      ZOO_AUTOPURGE_SNAPRETAINCOUNT: 3
    healthcheck:
      test: ["CMD-SHELL", "echo ruok | nc localhost 2181 | grep -q 'imok'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 5s
  xatu-server:
    profiles:
      - ""
    command: server --config /etc/xatu-server/config.yaml
    container_name: xatu-server
    hostname: xatu-server
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - "${XATU_SERVER_ADDRESS:-0.0.0.0}:${XATU_SERVER_PORT:-8080}:8080"
    environment:
      EVENT_INGESTER_AUTHORIZATION_SECRET: ${EVENT_INGESTER_AUTHORIZATION_SECRET}
    volumes:
      - ./deploy/local/docker-compose/xatu-server.yaml:/etc/xatu-server/config.yaml
    networks:
      - xatu-net

  xatu-postgres:
    profiles:
      - ""
    image: postgres:15
    container_name: xatu-postgres
    hostname: xatu-postgres
    volumes:
      - postgres-data:/var/lib/postgresql/data
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-xatu}
      POSTGRES_USER: ${POSTGRES_USER:-user}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-user} -d ${POSTGRES_DB:-xatu}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    networks:
      - xatu-net

  xatu-grafana:
    profiles:
      - ""
    image: grafana/grafana:latest
    container_name: xatu-grafana
    ports:
      - "${GRAFANA_ADDRESS:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000"
    networks:
      - xatu-net
    environment:
      GF_INSTALL_PLUGINS: grafana-clickhouse-datasource,vertamedia-clickhouse-datasource
      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/xatu/basic_overview.json
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
      GF_AUTH_ANONYMOUS_ORG_NAME: Main Org.
    volumes:
      - ./deploy/local/docker-compose/grafana/datasources:/etc/grafana/provisioning/datasources
      - ./deploy/local/docker-compose/grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml
      - ./deploy/local/docker-compose/grafana/dashboards:/var/lib/grafana/dashboards
  xatu-prometheus:
    profiles:
      - ""
    image: prom/prometheus:latest
    ports:
      - "${PROMETHEUS_ADDRESS:-127.0.0.1}:${PROMETHEUS_PORT:-9090}:9090"
    networks:
      - xatu-net
    volumes:
      - prometheus-data:/prometheus
      - ./deploy/local/docker-compose/prometheus.yaml:/etc/prometheus/prometheus.yml

  xatu-init-kafka:
    profiles:
      - ""
    image: confluentinc/cp-kafka:latest
    hostname: xatu-init-kafka
    container_name: xatu-init-kafka
    entrypoint:
      - bash
      - -c
    command:
      - |
        topics=(
          "beacon-api-eth-v1-beacon-blob-sidecar"
          "beacon-api-eth-v1-beacon-committee"
          "beacon-api-eth-v1-debug-fork-choice"
          "beacon-api-eth-v1-debug-fork-choice-reorg"
          "beacon-api-eth-v1-debug-fork-choice-reorg-v2"
          "beacon-api-eth-v1-debug-fork-choice-v2"
          "beacon-api-eth-v1-events-attestation"
          "beacon-api-eth-v1-events-attestation-v2"
          "beacon-api-eth-v1-events-blob-sidecar"
          "beacon-api-eth-v1-events-block"
          "beacon-api-eth-v1-events-block-v2"
          "beacon-api-eth-v1-events-chain-reorg"
          "beacon-api-eth-v1-events-chain-reorg-v2"
          "beacon-api-eth-v1-events-contribution-and-proof"
          "beacon-api-eth-v1-events-contribution-and-proof-v2"
          "beacon-api-eth-v1-events-finalized-checkpoint"
          "beacon-api-eth-v1-events-finalized-checkpoint-v2"
          "beacon-api-eth-v1-events-head"
          "beacon-api-eth-v1-events-head-v2"
          "beacon-api-eth-v1-events-voluntary-exit"
          "beacon-api-eth-v1-events-voluntary-exit-v2"
          "beacon-api-eth-v1-proposer-duty"
          "beacon-api-eth-v1-validator-attestation-data"
          "beacon-api-eth-v2-beacon-block"
          "beacon-api-eth-v2-beacon-block-attester-slashing"
          "beacon-api-eth-v2-beacon-block-bls-to-execution-change"
          "beacon-api-eth-v2-beacon-block-deposit"
          "beacon-api-eth-v2-beacon-block-execution-transaction"
          "beacon-api-eth-v2-beacon-block-proposer-slashing"
          "beacon-api-eth-v2-beacon-block-v2"
          "beacon-api-eth-v2-beacon-block-voluntary-exit"
          "beacon-api-eth-v2-beacon-block-withdrawal"
          "beacon-api-eth-v2-beacon-block-elaborated-attestation"
          "beacon-p2p-attestation"
          "blockprint-block-classification"
          "mempool-transaction"
          "mempool-transaction-v2"
          "libp2p-trace-connected"
          "libp2p-trace-disconnected"
          "libp2p-trace-add-peer"
          "libp2p-trace-remove-peer"
          "libp2p-trace-recv-rpc"
          "libp2p-trace-send-rpc"
          "libp2p-trace-join"
          "libp2p-trace-handle-metadata"
          "libp2p-trace-handle-status"
          "libp2p-trace-gossipsub-beacon-block"
          "libp2p-trace-gossipsub-beacon-attestation"
          "libp2p-trace-gossipsub-blob-sidecar"
          "beacon-api-eth-v1-beacon-validators"
          "mev-relay-bid-trace-builder-block-submission"
          "mev-relay-proposer-payload-delivered"
        )
        for topic in "$${topics[@]}"; do
          echo "Creating topic: $$topic";
          kafka-topics --create --if-not-exists --bootstrap-server xatu-kafka:29092 --partitions 1 --replication-factor 1 --config cleanup.policy=compact --config retention.ms=300000 --topic "$$topic"
        done

        sleep 3;
    depends_on:
      xatu-kafka:
        condition: service_healthy
    networks:
      - xatu-net

  xatu-kafka:
    profiles:
      - ""
    image: confluentinc/cp-kafka:latest
    hostname: xatu-kafka
    container_name: xatu-kafka
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: xatu-kafka-zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://xatu-kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_LOG4J_LOGGERS: "kafka.controller=ERROR,kafka.producer.async.DefaultEventHandler=ERROR,state.change.logger=ERROR"
    ports:
      - "${KAFKA_ADDRESS:-127.0.0.1}:${KAFKA_PORT:-29092}:29092"
      - "${KAFKA_BROKER_ADDRESS:-127.0.0.1}:${KAFKA_BROKER_PORT:-9092}:9092"
    networks:
      - xatu-net
    volumes:
      - kafka-data:/var/lib/kafka/data
    healthcheck:
      test:
        [
          "CMD",
          "kafka-broker-api-versions",
          "--bootstrap-server=localhost:9092",
        ]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 5s
    depends_on:
      xatu-kafka-zookeeper:
        condition: service_healthy
  xatu-kafka-zookeeper:
    profiles:
      - ""
    image: zookeeper
    container_name: xatu-kafka-zookeeper
    environment:
      ZOO_LOG4J_PROP: "ERROR,CONSOLE"
    ports:
      - "${KAFKA_ZOOKEEPER_ADDRESS:-127.0.0.1}:${KAFKA_ZOOKEEPER_PORT:-2181}:2181"
    networks:
      - xatu-net
    volumes:
      - kafka-zookeeper-data:/data
      - kafka-zookeeper-datalog:/datalog
    healthcheck:
      test: ["CMD", "./bin/zkServer.sh", "status"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 5s

  xatu-vector-http-kafka:
    profiles:
      - ""
    image: timberio/vector:0.34.1-alpine
    container_name: xatu-vector-http-kafka
    volumes:
      - ./deploy/local/docker-compose/vector-http-kafka.yaml:/etc/vector/vector.yaml
    environment:
      KAFKA_BROKERS: "xatu-kafka:29092"
    ports:
      - "${VECTOR_HTTP_KAFKA_ADDRESS:-127.0.0.1}:${VECTOR_HTTP_KAFKA_PORT:-9005}:9005"
    networks:
      - xatu-net
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8686/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 5s
    depends_on:
      xatu-kafka:
        condition: service_healthy
      xatu-init-kafka:
        condition: service_completed_successfully

  xatu-vector-kafka-clickhouse:
    profiles:
      - ""
    image: timberio/vector:0.37.1-alpine
    container_name: xatu-vector-kafka-clickhouse
    volumes:
      - ./deploy/local/docker-compose/vector-kafka-clickhouse.yaml:/etc/vector/vector.yaml
    networks:
      - xatu-net
    ports:
      - "${VECTOR_KAFKA_CLICKHOUSE_ADDRESS:-127.0.0.1}:${VECTOR_KAFKA_CLICKHOUSE_PORT:-8686}:8686"
    environment:
      CLICKHOUSE_ENDPOINT: "xatu-clickhouse-01:8123"
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      KAFKA_BROKERS: "xatu-kafka:29092"
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8686/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 5s
    depends_on:
      xatu-kafka:
        condition: service_healthy
      xatu-clickhouse-01:
        condition: service_healthy
      xatu-clickhouse-02:
        condition: service_healthy
      xatu-clickhouse-migrator:
        condition: service_completed_successfully
      xatu-init-kafka:
        condition: service_completed_successfully
  xatu-vector-kafka-clickhouse-libp2p:
    profiles:
      - ""
    image: timberio/vector:0.37.1-alpine
    volumes:
      - ./deploy/local/docker-compose/vector-kafka-clickhouse-libp2p.yaml:/etc/vector/vector.yaml
    networks:
      - xatu-net
    environment:
      CLICKHOUSE_ENDPOINT: "xatu-clickhouse-01:8123"
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      KAFKA_BROKERS: "xatu-kafka:29092"
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8686/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 5s
    depends_on:
      xatu-kafka:
        condition: service_healthy
      xatu-clickhouse-01:
        condition: service_healthy
      xatu-clickhouse-02:
        condition: service_healthy
      xatu-clickhouse-migrator:
        condition: service_completed_successfully
      xatu-init-kafka:
        condition: service_completed_successfully
  xatu-postgres-migrator:
    profiles:
      - ""
    image: migrate/migrate
    container_name: xatu-postgres-migrator
    volumes:
      - ./migrations/postgres:/migrations
    command:
      [
        "-path",
        "/migrations",
        "-database",
        "postgres://user:password@xatu-postgres:5432/xatu?sslmode=disable",
        "up",
      ]
    depends_on:
      xatu-postgres:
        condition: service_healthy
    networks:
      - xatu-net
  xatu-clickhouse-migrator:
    profiles:
      - clickhouse
      - ""
    image: migrate/migrate
    container_name: xatu-clickhouse-migrator
    volumes:
      - ./deploy/migrations/clickhouse:/migrations
    command:
      [
        "-path",
        "/migrations",
        "-database",
        "clickhouse://xatu-clickhouse-01:9000?username=default&database=default&x-multi-statement=true",
        "up",
      ]
    depends_on:
      xatu-clickhouse-01:
        condition: service_healthy
      xatu-clickhouse-02:
        condition: service_healthy
    networks:
      - xatu-net
  tempo-init:
    image: &tempoImage grafana/tempo:latest
    user: root
    entrypoint:
      - "chown"
      - "10001:10001"
      - "/var/tempo"
    volumes:
      - tempo-data:/var/tempo
  tempo:
    image: *tempoImage
    container_name: xatu-tempo
    command: [ "-config.file=/etc/tempo.yaml" ]
    volumes:
      - ./deploy/local/docker-compose/tempo.yaml:/etc/tempo.yaml
      - tempo-data:/var/tempo
    ports:
      - "${TEMPO_JAEGER_ADDRESS:-127.0.0.1}:${TEMPO_JAEGER_PORT:-6831}:6831/udp"  # jaeger ingest
      - "${TEMPO_HTTP_ADDRESS:-127.0.0.1}:${TEMPO_HTTP_PORT:-3200}:3200"   # tempo
      - "${TEMPO_GRPC_ADDRESS:-127.0.0.1}:${TEMPO_GRPC_PORT:-9095}:9095" # tempo grpc
      - "${TEMPO_OTLP_GRPC_ADDRESS:-127.0.0.1}:${TEMPO_OTLP_GRPC_PORT:-4317}:4317"  # otlp grpc
      - "${TEMPO_OTLP_HTTP_ADDRESS:-127.0.0.1}:${TEMPO_OTLP_HTTP_PORT:-4318}:4318"  # otlp http
      - "${TEMPO_ZIPKIN_ADDRESS:-127.0.0.1}:${TEMPO_ZIPKIN_PORT:-9411}:9411"   # zipkin
    networks:
      - xatu-net
    depends_on:
      - tempo-init
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3200/ready"]
      interval: 5s
      timeout: 5s
      retries: 5
  xatu-nginx:
    profiles:
      - ""
    image: nginx:1.27.1-bookworm
    container_name: xatu-nginx
    ports:
      - "${NGINX_ADDRESS:-127.0.0.1}:${NGINX_PORT:-8044}:80"
    environment:
      - BASE_HOSTNAME=${BASE_HOSTNAME:-example.com}
    volumes:
      - ./deploy/local/docker-compose/nginx/nginx.conf:/tmp/nginx.conf:ro
    networks:
      - xatu-net
    command: >
      /bin/sh -c "
        sed 's/\$$HOSTNAME/'"$$BASE_HOSTNAME"'/g' /tmp/nginx.conf > /etc/nginx/nginx.conf &&
        nginx -g 'daemon off;'
      "

networks:
  xatu-net:
    driver: bridge

volumes:
  postgres-data:
    driver: local
  clickhouse-01-data:
    driver: local
  clickhouse-02-data:
    driver: local
  clickhouse-zookeeper-01-data:
    driver: local
  clickhouse-zookeeper-02-data:
    driver: local
  clickhouse-zookeeper-03-data:
    driver: local
  clickhouse-zookeeper-01-datalog:
    driver: local
  clickhouse-zookeeper-02-datalog:
    driver: local
  clickhouse-zookeeper-03-datalog:
    driver: local
  kafka-data:
    driver: local
  kafka-zookeeper-data:
    driver: local
  kafka-zookeeper-datalog:
    driver: local
  prometheus-data:
    driver: local
  tempo-data:
    driver: local