Skip to content

Commit

Permalink
Use docker-compose for Github CI workflow
Browse files Browse the repository at this point in the history
It's bugging me that we now have both a docker-compose.yml and an
`integration-tests` section in ci.yaml. The two are identical, but different
syntax. I've updated the Github workflow to use `docker compose` instead of
duplicating the service logic.
  • Loading branch information
criccomini committed Jan 26, 2024
1 parent a431858 commit 6c1428e
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 101 deletions.
129 changes: 28 additions & 101 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ name: Test
on:
push:
branches: [ main ]

pull_request:
branches: [ main ]

Expand Down Expand Up @@ -57,98 +56,18 @@ jobs:
runs-on: ubuntu-latest
needs: unit-tests

services:
postgres:
image: postgres:14
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
POSTGRES_DB: testdb
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
mysql:
image: mysql:8.0
env:
MYSQL_ROOT_PASSWORD: password
MYSQL_DATABASE: testdb
MYSQL_USER: mysql
MYSQL_PASSWORD: password
ports:
- 3306:3306
options: >-
--health-cmd "mysqladmin ping --silent"
--health-interval 10s
--health-timeout 5s
--health-retries 5
# TODO Waiting on https://github.com/goccy/bigquery-emulator/pull/208
bigquery:
image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar
env:
BIGQUERY_EMULATOR_PROJECT: test_project
BIGQUERY_EMULATOR_DATASET: test_dataset
ports:
- 9050:9050

zookeeper:
image: confluentinc/cp-zookeeper:latest
ports:
- 2181:2181
env:
ZOOKEEPER_CLIENT_PORT: 2181
options: >-
--health-cmd "echo ruok | nc localhost 2181"
--health-interval 10s
--health-timeout 5s
--health-retries 5
kafka:
image: confluentinc/cp-kafka:latest
ports:
- 9092:9092
env:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
options: >-
--health-cmd "kafka-topics --bootstrap-server localhost:9092 --list"
--health-interval 10s
--health-timeout 5s
--health-retries 5
schema-registry:
image: confluentinc/cp-schema-registry:latest
ports:
- 8081:8081
env:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
options: >-
--health-cmd "curl -f http://schema-registry:8081/"
--health-interval 10s
--health-timeout 5s
--health-retries 5
hive-metastore:
image: ghcr.io/criccomini/hive-metastore-standalone:latest
ports:
- 9083:9083

strategy:
matrix:
python-version: ['3.10']

steps:
- uses: actions/checkout@v3

- name: Docker Compose Up
uses: isbang/compose-action@v1.5.1
with:
compose-file: tests/docker-compose.yml

- name: Build using Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
Expand All @@ -166,25 +85,33 @@ jobs:
run: |
pdm run integration
- name: Docker Compose Down
uses: isbang/compose-action@v1.5.1
with:
compose-file: tests/docker-compose.yml

spec-tests:
runs-on: ubuntu-latest
needs: integration-tests

strategy:
matrix:
python-version: ['3.10']

steps:
- uses: actions/checkout@v3

- name: Build using Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
run: |
pip3 install pdm
pdm self update
pdm install
- name: Test spec with pytest
run: |
pdm run spec
- uses: actions/checkout@v3

- name: Build using Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies
run: |
pip3 install pdm
pdm self update
pdm install
- name: Test spec with pytest
run: |
pdm run spec
1 change: 1 addition & 0 deletions tests/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ services:
retries: 5

bigquery:
# TODO Waiting on https://github.com/goccy/bigquery-emulator/pull/208
image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar
environment:
BIGQUERY_EMULATOR_PROJECT: test_project
Expand Down

0 comments on commit 6c1428e

Please sign in to comment.