From 4b2565cffa3431d5d9d5be862a7e3fb7ebea02d8 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 21 Dec 2023 21:14:39 +0000 Subject: [PATCH] Add a check for Jenkins results. Skip-build: true Required-githooks: true Signed-off-by: Ashley Pittman --- .github/workflows/bash_unit_testing.yml | 1 + .github/workflows/jenkins-status.yml | 20 +++ .github/workflows/linting.yml | 5 +- ci/daily_status.py | 188 ++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/jenkins-status.yml create mode 100755 ci/daily_status.py diff --git a/.github/workflows/bash_unit_testing.yml b/.github/workflows/bash_unit_testing.yml index 7f8ae63f43bc..173ca16c3211 100644 --- a/.github/workflows/bash_unit_testing.yml +++ b/.github/workflows/bash_unit_testing.yml @@ -15,6 +15,7 @@ defaults: jobs: Test-gha-functions: name: Tests in ci/gha_functions.sh + if: github.repository == 'daos-stack/daos' runs-on: [self-hosted, light] steps: - name: Checkout code diff --git a/.github/workflows/jenkins-status.yml b/.github/workflows/jenkins-status.yml new file mode 100644 index 000000000000..22f1f038122f --- /dev/null +++ b/.github/workflows/jenkins-status.yml @@ -0,0 +1,20 @@ +name: Jenkins status report + +on: + issue_comment: + types: [created] + +jobs: + # Check and report Jenkins test results. Should use the check_suite trigger when stable, and + # test the PR that triggered it obviously. + jenkins_check: + name: Check Jenkins results + if: github.repository == 'daos-stack/daos' and ${{ github.event.issue.pull_request }} + runs-on: [self-hosted, light] + steps: + - uses: actions/checkout@v4 + - name: Run check + run: \[ ! -x ci/daily_status.py \] || ./ci/daily_status.py --pr $NUMBER + env: + NUMBER: ${{ github.event.issue.number }} + NAME: ${{ github.event.sender}} diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 9bdef7c96950..149d509abb0e 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -43,10 +43,7 @@ jobs: - name: Add error parser run: echo -n "::add-matcher::ci/shellcheck-matcher.json" - name: Run Shellcheck - # The check will run with this file from the target branch but the code from the PR so - # test for this file before calling it to prevent failures on PRs where this check is - # in the target branch but the PR is not updated to include it. - run: \[ ! -x ci/run_shellcheck.sh \] || ./ci/run_shellcheck.sh + run: ./ci/run_shellcheck.sh log-check: name: Logging macro checking diff --git a/ci/daily_status.py b/ci/daily_status.py new file mode 100755 index 000000000000..e3b657a0a803 --- /dev/null +++ b/ci/daily_status.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +"""Parse Jenkins build test results""" + +import argparse +import json +import urllib +from urllib.request import urlopen + +JENKINS_HOME = "https://build.hpdd.intel.com/job/daos-stack" + + +class TestResult: + """Represents a single Jenkins test result""" + + def __init__(self, data, blocks): + name = data["name"] + if "-" in name: + try: + (num, full) = name.split("-", 1) + test_num = int(num) + if full[-5] == "-": + full = full[:-5] + name = f"{full} ({test_num})" + except ValueError: + pass + self.name = name + self.cname = data["className"] + self.skipped = False + self.passed = False + self.failed = False + assert data["status"] in ("PASSED", "FIXED", "SKIPPED", "FAILED", "REGRESSION") + if data["status"] in ("PASSED", "FIXED"): + self.passed = True + elif data["status"] in ("FAILED", "REGRESSION"): + self.failed = True + elif data["status"] == "SKIPPED": + self.skipped = True + self.data = data + self.blocks = blocks + + def info(self, prefix=""): + """Return a string describing the test""" + return f"{prefix}{self.cname}\t\t{self.name}" + + def full_info(self): + """Return a longer string describing the test""" + + tcl = [] + if self.blocks is not None: + tcl.extend(reversed(self.blocks)) + + tcl.append(f"{self.cname}.{self.name}") + details = self.data["errorDetails"] + if details: + return " / ".join(tcl) + "\n" + details.replace("\\n", "\n") + return self.info() + + # Needed for set operations to compare results across sets. + def __eq__(self, other): + return self.name == other.name and self.cname == other.cname + + # Needed to be able to add results to sets. + def __hash__(self): + return hash((self.name, self.cname)) + + def __str__(self): + return self.name + + def __repr__(self): + return f"Test result of {self.cname}" + + +def je_load(job_name="daily-testing", jid=None, what=None, tree=None): + """Fetch something from Jenkins and return as native type.""" + if jid: + if what: + url = f"{JENKINS_HOME}/job/daos/job/{job_name}/{jid}/{what}/api/json" + else: + url = f"{JENKINS_HOME}/job/daos/job/{job_name}/{jid}/api/json" + else: + url = f"{JENKINS_HOME}/job/daos/job/{job_name}/api/json" + + if tree: + url += f"?tree={tree}" + + with urlopen(url) as f: + return json.load(f) + + +def show_job(jid, job_name="daily-testing"): + """Show one job""" + + if not job_name.startswith("PR-"): + jdata = je_load(job_name=job_name, jid=jid, tree="actions[causes]") + if ( + "causes" not in jdata["actions"][0] + or jdata["actions"][0]["causes"][0]["_class"] + != "hudson.triggers.TimerTrigger$TimerTriggerCause" + ): + return None + + try: + jdata = je_load(job_name=job_name, jid=jid, what="testReport") + except urllib.error.HTTPError: + return None + + failed = [] + + assert not jdata["testActions"] + for suite in jdata["suites"]: + for k in suite["cases"]: + tr = TestResult(k, suite["enclosingBlockNames"]) + if not tr.failed: + continue + failed.append(tr) + return failed + + +def main(): + """Check the results of a PR""" + + parser = argparse.ArgumentParser(description="Check Jenkins test results") + parser.add_argument("--pr", type=int, required=True) + + args = parser.parse_args() + + job_name = f"PR-{args.pr}" + + data = je_load(job_name=job_name) + + lcb = data["lastCompletedBuild"]["number"] + + all_failed = set() + for build in data["builds"]: + jid = build["number"] + if jid > lcb: + print(f"Job {jid} is of {job_name} is still running, skipping") + continue + failed = show_job(job_name=job_name, jid=jid) + if not isinstance(failed, list): + continue + for test in failed: + all_failed.add(test) + break + if not all_failed: + print("No failed tests in PR, returning") + + print("PR had failed tests, checking against landings builds") + + job_name = "daily-testing" + data = je_load(job_name=job_name) + lcb = data["lastCompletedBuild"]["number"] + main_failed = set() + check_count = 0 + for build in data["builds"]: + jid = build["number"] + if jid > lcb: + print(f"Job {jid} is of {job_name} is still running, skipping") + failed = show_job(job_name=job_name, jid=jid) + if not isinstance(failed, list): + continue + for test in failed: + main_failed.add(test) + check_count += 1 + if check_count > 14: + break + + unexplained = all_failed.difference(main_failed) + if not unexplained: + print(f"Stopping checking at {check_count} builds, all failures explained") + break + + ignore = all_failed.intersection(main_failed) + if ignore: + print("Tests which failed in the PR and have also failed in landings builds.") + for test in ignore: + print(test.full_info()) + + new = all_failed.difference(main_failed) + if new: + print("Tests which only failed in the PR") + for test in new: + print(test.full_info()) + + +if __name__ == "__main__": + main()