Add BigQueryReader.to_recap #292
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: recap | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| jobs: | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ['3.10'] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Build using Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python3.10 -m venv venv | |
| source venv/bin/activate | |
| pip install pdm | |
| pdm install | |
| - name: Run black | |
| run: | | |
| source venv/bin/activate | |
| pdm run black --check --diff recap/ tests/ | |
| - name: Run isort | |
| run: | | |
| source venv/bin/activate | |
| pdm run isort recap/ tests/ --check-only --diff | |
| - name: Run pylint | |
| run: | | |
| source venv/bin/activate | |
| pdm run pylint --fail-under=7.0 recap/ tests/ | |
| - name: Run pyright | |
| run: | | |
| source venv/bin/activate | |
| pdm run pyright | |
| - name: Test with pytest | |
| run: | | |
| source venv/bin/activate | |
| pdm run pytest tests/unit -vv | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| needs: unit-tests | |
| services: | |
| postgres: | |
| image: postgres:14 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: password | |
| POSTGRES_DB: testdb | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| # TODO Waiting on https://github.com/goccy/bigquery-emulator/pull/208 | |
| bigquery: | |
| image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar | |
| env: | |
| BIGQUERY_EMULATOR_PROJECT: test_project | |
| BIGQUERY_EMULATOR_DATASET: test_dataset | |
| ports: | |
| - 9050:9050 | |
| zookeeper: | |
| image: confluentinc/cp-zookeeper:latest | |
| ports: | |
| - 2181:2181 | |
| env: | |
| ZOOKEEPER_CLIENT_PORT: 2181 | |
| options: >- | |
| --health-cmd "echo ruok | nc localhost 2181" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| kafka: | |
| image: confluentinc/cp-kafka:latest | |
| ports: | |
| - 9092:9092 | |
| env: | |
| KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 | |
| KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 | |
| KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | |
| options: >- | |
| --health-cmd "kafka-topics --bootstrap-server localhost:9092 --list" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| schema-registry: | |
| image: confluentinc/cp-schema-registry:latest | |
| ports: | |
| - 8081:8081 | |
| env: | |
| SCHEMA_REGISTRY_HOST_NAME: schema-registry | |
| SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081 | |
| SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 | |
| SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092 | |
| options: >- | |
| --health-cmd "curl -f http://schema-registry:8081/" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| hive-metastore: | |
| image: ghcr.io/criccomini/hive-metastore-standalone:latest | |
| ports: | |
| - 9083:9083 | |
| strategy: | |
| matrix: | |
| python-version: ['3.10'] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Build using Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python3.10 -m venv venv | |
| source venv/bin/activate | |
| pip install pdm | |
| pdm install | |
| - name: Test with pytest | |
| run: | | |
| source venv/bin/activate | |
| pdm run pytest tests/integration -vv |