-
Notifications
You must be signed in to change notification settings - Fork 24
169 lines (142 loc) · 4.03 KB
/
ci.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
name: recap
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
unit-tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.10']
steps:
- uses: actions/checkout@v2
- name: Build using Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip3 install pdm
pdm self update
pdm install
- name: Run black
run: |
pdm run black --check --diff recap/ tests/
- name: Run isort
run: |
pdm run isort recap/ tests/ --check-only --diff
- name: Run pylint
run: |
pdm run pylint --fail-under=7.0 recap/ tests/
- name: Run pyright
run: |
pdm run pyright
- name: Test with pytest
run: |
pdm run unit
integration-tests:
runs-on: ubuntu-latest
needs: unit-tests
services:
postgres:
image: postgres:14
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
POSTGRES_DB: testdb
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
# TODO Waiting on https://github.com/goccy/bigquery-emulator/pull/208
bigquery:
image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar
env:
BIGQUERY_EMULATOR_PROJECT: test_project
BIGQUERY_EMULATOR_DATASET: test_dataset
ports:
- 9050:9050
zookeeper:
image: confluentinc/cp-zookeeper:latest
ports:
- 2181:2181
env:
ZOOKEEPER_CLIENT_PORT: 2181
options: >-
--health-cmd "echo ruok | nc localhost 2181"
--health-interval 10s
--health-timeout 5s
--health-retries 5
kafka:
image: confluentinc/cp-kafka:latest
ports:
- 9092:9092
env:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
options: >-
--health-cmd "kafka-topics --bootstrap-server localhost:9092 --list"
--health-interval 10s
--health-timeout 5s
--health-retries 5
schema-registry:
image: confluentinc/cp-schema-registry:latest
ports:
- 8081:8081
env:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
options: >-
--health-cmd "curl -f http://schema-registry:8081/"
--health-interval 10s
--health-timeout 5s
--health-retries 5
hive-metastore:
image: ghcr.io/criccomini/hive-metastore-standalone:latest
ports:
- 9083:9083
strategy:
matrix:
python-version: ['3.10']
steps:
- uses: actions/checkout@v2
- name: Build using Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip3 install pdm
pdm self update
pdm install
- name: Test with pytest
run: |
pdm run integration
spec-tests:
runs-on: ubuntu-latest
needs: integration-tests
strategy:
matrix:
python-version: ['3.10']
steps:
- uses: actions/checkout@v2
- name: Build using Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip3 install pdm
pdm self update
pdm install
- name: Test spec with pytest
run: |
pdm run spec