diff --git a/DEVELOPER.md b/DEVELOPER.md index f96d5f0..ab16cc3 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -30,9 +30,13 @@ These tests require internet access because they download the spec from Recap's Recap has integration tests that validate some of Recap's readers against real systems. These tests are located in the `tests/integration` directory. +These tests require a postgres and mysql database to be running to work. To spin them up, you can use docker-compose: + + docker-compose --project-directory tests/ up + Run the integration tests: - pdm run integration + RECAP_URLS='["postgresql://postgres:password@localhost:5432/testdb"]' pdm run integration These test require various systems to be running as defined in the .github/workflows/ci.yml file. diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..230f734 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,90 @@ +version: '3.8' + +services: + postgres: + image: postgres:14 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: testdb + ports: + - "5432:5432" + healthcheck: + test: ["CMD", "pg_isready"] + interval: 10s + timeout: 5s + retries: 5 + + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: testdb + MYSQL_USER: mysql + MYSQL_PASSWORD: password + ports: + - "3306:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "--silent"] + interval: 10s + timeout: 5s + retries: 5 + + bigquery: + image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar + environment: + BIGQUERY_EMULATOR_PROJECT: test_project + BIGQUERY_EMULATOR_DATASET: test_dataset + ports: + - "9050:9050" + + zookeeper: + image: confluentinc/cp-zookeeper:latest + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + healthcheck: + test: ["CMD", "echo", "ruok", "|", "nc", "localhost", "2181"] + interval: 10s + timeout: 5s + retries: 5 + + kafka: + image: confluentinc/cp-kafka:latest + ports: + - "9092:9092" + environment: + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + depends_on: + - zookeeper + healthcheck: + test: ["CMD", "kafka-topics", "--bootstrap-server", "localhost:9092", "--list"] + interval: 10s + timeout: 5s + retries: 5 + + schema-registry: + image: confluentinc/cp-schema-registry:latest + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081 + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092 + depends_on: + - kafka + - zookeeper + healthcheck: + test: ["CMD-SHELL", "curl -f http://schema-registry:8081/"] + interval: 10s + timeout: 5s + retries: 5 + + hive-metastore: + image: ghcr.io/criccomini/hive-metastore-standalone:latest + ports: + - "9083:9083"