-
Notifications
You must be signed in to change notification settings - Fork 480
160 lines (133 loc) · 7.39 KB
/
parallel_tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
name: Parallel Tests
# For PRs to dev or pushes that modify the root Dockerfile, build from scratch
# then run CI tests using that container in parallel
# For forked repos that can't use our self-hosted test suite, just build and run make check
on:
pull_request:
branches:
- dev
- stable
- candidate_release_*
#push:
# paths: ['Dockerfile'] # If this file changed, we'd need to do a clean build (this action)
# otherwise we could speed this up by pulling the last container of 'dev', copying
# code into it, and then rebuilding
jobs:
test_installer: # test install_ubuntu.sh
runs-on: ubuntu-20.04 # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet)
steps:
- uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
- name: Lint PyPANDA with flake8
run: |
python -m pip install --upgrade pip
python -m pip install flake8
python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics
# python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Run install_ubuntu.sh
run: cd $GITHUB_WORKSPACE && ./panda/scripts/install_ubuntu.sh
build_container:
if: github.repository == 'panda-re/panda'
runs-on: self-hosted
steps:
- uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
- name: Build docker container from project root
run: cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local_${{ github.sha }} .
- name: Minimal test of built container # Just test to see if one of our binaries is built
run: docker run --rm "panda_local_${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")'
taint_tests:
if: github.repository == 'panda-re/panda'
runs-on: self-hosted
needs: [build_container]
strategy:
matrix:
target: [i386, x86_64]
steps:
# Given a container with PANDA installed at /panda, run the taint tests
- name: Run taint tests inside current container
run: >-
docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID}
--mount type=bind,source=/home/panda/regdir/qcows/wheezy_panda2.qcow2,target=/home/panda/regdir/qcows/wheezy_panda2.qcow2
--mount type=bind,source=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2,target=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2
--rm -t "panda_local_${{ github.sha }}" bash -c
"cd /tmp; git clone https://github.com/panda-re/panda_test;
cd ./panda_test/tests/taint2;
python3 taint2_multi_arch_record_or_replay.py --arch ${{ matrix.target }} --mode record;
python3 taint2_multi_arch_record_or_replay.py --arch ${{ matrix.target }} --mode replay;
sed -i '/^\s*$/d' taint2_log;
if cat taint2_log; then echo 'Taint unit test log found!'; else echo 'Taint unit test log NOT found!' && exit 1; fi;
echo -e '\nFailures:';
if grep 'fail' taint2_log; then echo 'TEST FAILED!' && exit 1; else echo -e 'None.\nTEST PASSED!' && exit 0; fi"
sym_trace_tests:
if: github.repository == 'panda-re/panda'
runs-on: self-hosted
needs: [build_container]
strategy:
matrix:
target: [x86_64]
steps:
# Given a container with PANDA installed at /panda, run the taint tests
- name: Run symbolic tracing tests inside current container
run: >-
docker run --name panda_sym_test_${{ matrix.target }}_${GITHUB_RUN_ID}
--rm -t "panda_local_${{ github.sha }}" bash -c
"pip3 install capstone keystone-engine z3-solver; python3 /panda/panda/python/examples/unicorn/taint_sym_x86_64.py;
if [ $? -eq 0 ]; then echo -e 'TEST PASSED!' && exit 0; else echo 'TEST FAILED!' && exit 1; fi"
make_check:
if: github.repository == 'panda-re/panda'
runs-on: self-hosted
needs: [build_container]
strategy:
matrix:
# See output from `make check-help`: we're just splitting `make check` into all the things it does
# so we can run them in parallel: arch-specific qtests, plus a few others
target: [check-qtest-x86_64, check-qtest-i386, check-qtest-arm, check-qtest-mips, check-qtest-mipsel, check-qtest-ppc, check-block, check-unit, check-qapi-schema]
steps:
- name: Run Individual QEMU tests
run: >-
docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID}
-e PANDA_TEST=yes --cap-add SYS_NICE
--rm -t "panda_local_${{ github.sha }}" bash -c
"cd /panda/build && make ${{ matrix.target }}"
pypanda_tests:
if: github.repository == 'panda-re/panda'
runs-on: self-hosted
needs: [build_container]
strategy:
matrix:
# See output from `make check-help`: we're just splitting `make check` into all the things it does
# so we can run them in parallel: arch-specific qtests, plus a few others
test_script: [dyn_hooks, copy_test, file_fake, file_hook, generic_tests, monitor_cmds, multi_proc_cbs, sleep_in_cb, syscalls, record_no_snap, sig_suppress]
steps:
- name: Run individual pypanda tests
# TODO: pip requirements install here should be moved to Docker image build to save test time
run: >-
docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID}
--mount type=bind,source=/home/panda/regdir/qcows/ubuntu_1604_x86.qcow,target=/root/.panda/ubuntu_1604_x86.qcow
-e PANDA_TEST=yes --cap-add SYS_NICE
--rm -t "panda_local_${{ github.sha }}" bash -c
"cd /panda/panda/python/tests/ && make && pip3 install -r requirements.txt && python3 ${{ matrix.test_script }}.py"
cleanup:
# Cleanup after prior jobs finish - even if they fail
needs: [taint_tests, sym_trace_tests, make_check, pypanda_tests]
runs-on: self-hosted
if: always()
steps:
# Note we leave the last 72hrs because caching is nice (first few panda image layers won't change often)
# docker system prune -> Remove all unused containers, networks, images (both dangling and unreferenced)
# docker builder prune -> Remove build cache
- name: Cleanup images
run: |
docker system prune -af --filter "until=72h"
docker image prune --all -f --filter "until=72h"
docker builder prune -af --filter "until=72h"
build_and_check_fork: # Forked repos can't use self-hosted test suite - just checkout and run make check
if: github.repository != 'panda-re/panda'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1 # Clones code into to /home/runner/work/panda
- name: Build docker container from project root
run: cd $GITHUB_WORKSPACE && docker build -t panda_local .
- name: Minimal test of built container # Just test to see if one of our binaries is installed
run: docker run --rm panda_local /bin/bash -c 'exit $(panda-system-arm -help | grep -q "usage. panda-system-arm")'
- name: Minimal test of built container # Run make check to check all architectures (in serial)
run: docker run --rm panda_local /bin/bash -c 'cd /panda/build && make check'