-
Notifications
You must be signed in to change notification settings - Fork 44
217 lines (189 loc) · 8.09 KB
/
run-unittests-py38-cov-report.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
name: "[Py3.8][COV REPORT] - All Unit Tests"
on:
workflow_dispatch:
pull_request:
branches: [ "main" ]
paths:
- "ads/**"
- "!ads/opctl/operator/**"
- "!ads/feature_store/**"
- "pyproject.toml"
# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
pull-requests: write
# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
jobs:
test:
name: python 3.8, ${{ matrix.name }}
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
name: ["unitary", "slow_tests"]
include:
- name: "unitary"
test-path: "tests/unitary"
# `model` tests running in "slow_tests",
# `feature_store` tests has its own test suite
# `forecast` tests has its own test suite
# 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
ignore-path: |
--ignore tests/unitary/with_extras/model \
--ignore tests/unitary/with_extras/feature_store \
--ignore tests/unitary/with_extras/operator/feature-store \
--ignore tests/unitary/with_extras/operator/forecast \
--ignore tests/unitary/with_extras/hpo
- name: "slow_tests"
test-path: "tests/unitary/with_extras/model"
steps:
- uses: actions/checkout@v4
- uses: ./.github/workflows/create-more-space
name: "Create more disk space"
- uses: actions/setup-python@v5
with:
python-version: "3.8"
cache: "pip"
cache-dependency-path: |
pyproject.toml
"**requirements.txt"
- uses: ./.github/workflows/set-dummy-conf
name: "Test config setup"
- uses: ./.github/workflows/test-env-setup
name: "Test env setup"
timeout-minutes: 30
# Installing pii deps for python3.8 test setup only, it will not work with python3.9/3.10, because
# 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
- name: "Install PII dependencies"
run: |
pip install -e ".[pii]"
- name: "Install featurestore marketplace dependencies"
run: |
pip install -e ".[feature-store-marketplace]"
- name: "Run unitary tests folder with maximum ADS dependencies"
timeout-minutes: 60
shell: bash
env:
CONDA_PREFIX: /usr/share/miniconda
run: |
set -x # print commands that are executed
# Setup project and tests folder for cov reports to not be overwritten by another parallel step
mkdir -p cov-${{ matrix.name }}
cd cov-${{ matrix.name }}
ln -s ../tests tests
ln -s ../ads ads
ln -s ../.coveragerc .coveragerc
# Run hpo tests, which hangs if run together with all unitary tests
if [[ ${{ matrix.name }} == "unitary" ]]; then
python -m pytest -v -p no:warnings -n auto --dist loadfile \
--cov-append --cov=ads --cov-report=xml --cov-report=html \
tests/unitary/with_extras/hpo
fi
# Run tests
python -m pytest -v -p no:warnings --durations=5 \
-n auto --dist loadfile \
--cov-append --cov=ads --cov-report=xml --cov-report=html \
${{ matrix.test-path }} ${{ matrix.ignore-path }}
- name: "Save coverage files"
uses: actions/upload-artifact@v4
with:
name: cov-reports-${{ matrix.name }}
path: |
cov-${{ matrix.name }}/htmlcov/
cov-${{ matrix.name }}/.coverage
cov-${{ matrix.name }}/coverage.xml
include-hidden-files: true
coverage-report:
name: "Coverage report"
runs-on: ubuntu-latest
continue-on-error: true
needs: test
if: ${{ success() }} && github.event_name == 'pull_request'
env:
COMPARE_BRANCH: main
steps:
- name: "Checkout current branch"
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Download coverage files"
uses: actions/download-artifact@v4
- name: "Calculate overall coverage"
run: |
set -x # print commands that are executed
# Prepare default cov body text
COV_BODY_INTRO="📌 Overall coverage:\n\n"
echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
# Prepare file paths to .coverage files
# Filenames taken from job.test last step with name - "Save coverage files"
FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
FILE_SLOW="cov-reports-slow_tests/.coverage"; [[ ! -f $FILE_SLOW ]] && FILE_SLOW=""
# Combine coverage files
pip install coverage
coverage combine $FILE_UNITARY $FILE_SLOW
# Make html report
coverage html
# Calculate overall coverage and update body message
COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
if [[ ! -z $COV ]]; then
if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
echo COV_BODY="$COV_BODY_INTRO " >> $GITHUB_ENV
fi
- name: "Calculate coverage diff"
run: |
set -x # print commands that are executed
# Prepare default diff body text
DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
# Prepare file paths to coverage xml files
# Filenames taken from job.test last step with name - "Save coverage files"
FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
FILE2="cov-reports-slow_tests/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
echo "FILE1=$FILE1" >> $GITHUB_ENV
echo "FILE2=$FILE2" >> $GITHUB_ENV
# Calculate coverage diff and update body message
pip install diff_cover
diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
DIFF=$(diff-cover $FILE1 $FILE2 \
--compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep Coverage: | cut -d' ' -f 2 | cut -d'%' -f 1)
if [[ -z $DIFF ]]; then
DIFF_INFO=$(diff-cover $FILE1 $FILE2 \
--compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep "No lines");
echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV
else
if [[ $DIFF -lt 50 ]]; then COLOR=red; elif [[ $DIFF -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
echo DIFF_BODY="$DIFF_BODY_INTRO " >> $GITHUB_ENV
fi
- name: "Add comment with cov diff to PR"
continue-on-error: true
uses: actions/github-script@v7
if: github.event_name == 'pull_request'
with:
github-token: ${{ github.token }}
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: '${{ env.DIFF_BODY }}\n\n${{ env.COV_BODY }}'
})
- name: "Generate html difference report"
run: |
diff-cover ${{ env.FILE1 }} ${{ env.FILE2 }} \
--compare-branch=origin/${{ env.COMPARE_BRANCH }} \
--html-report=cov-diff.html
- name: "Save coverage difference report"
uses: actions/upload-artifact@v4
with:
name: cov-html-reports
path: |
cov-diff.html
htmlcov/
retention-days: 10