From 1b920f543012bebcc3c2e48a7e653567896fe60d Mon Sep 17 00:00:00 2001 From: Thomas Carmet Date: Mon, 25 Apr 2022 13:10:21 -0700 Subject: [PATCH] BERTE-565 BERTE-566 handle status check when workflow is pending, remove workflow dispatch checks --- .github/workflows/main.yaml | 4 + CHANGELOG | 5 ++ bert_e/git_host/github/__init__.py | 115 ++++++++++++++++++++----- bert_e/git_host/github/schema.py | 23 +++++ eve/main.yml | 99 --------------------- eve/worker/dockerfile/Dockerfile | 74 ---------------- eve/worker/dockerfile/requirements.txt | 20 ----- eve/worker/worker.yaml | 23 ----- requirements.txt | 1 + 9 files changed, 126 insertions(+), 238 deletions(-) delete mode 100644 eve/main.yml delete mode 100644 eve/worker/dockerfile/Dockerfile delete mode 100644 eve/worker/dockerfile/requirements.txt delete mode 100644 eve/worker/worker.yaml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index db7065c1..be1a0421 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -64,6 +64,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - run: git config --global --add safe.directory /__w/bert-e/bert-e - run: tox -e utests - run: tox -e tests-api-mock - run: tox -e coverage-report @@ -79,6 +80,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - run: git config --global --add safe.directory /__w/bert-e/bert-e - run: tox -e tests-noqueue all-tests: @@ -91,6 +93,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - run: git config --global --add safe.directory /__w/bert-e/bert-e - run: tox -e tests lint: @@ -103,5 +106,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - run: git config --global --add safe.directory /__w/bert-e/bert-e - run: tox -e flake8 - run: helm lint charts/bert-e diff --git a/CHANGELOG b/CHANGELOG index 3d453268..bd56f149 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,11 @@ # Change Log All notable changes to this project will be documented in this file. +## [3.6.9] - 2022-04-25 +# Fixed +- Handle GitHub Actions status when workflow is pending. +- Don't check workflow triggerd by "workflow_dispatch" event + ## [3.6.8] - 2021-11-09 # Fixed - Allways get the latest check-runs link status diff --git a/bert_e/git_host/github/__init__.py b/bert_e/git_host/github/__init__.py index be30f651..d7ccf4ac 100644 --- a/bert_e/git_host/github/__init__.py +++ b/bert_e/git_host/github/__init__.py @@ -355,10 +355,9 @@ def get_commit_status(self, ref): combined = AggregatedStatus.get(self.client, owner=self.owner, repo=self.slug, ref=ref) - - actions = AggregatedCheckRuns.get(self.client, - owner=self.owner, - repo=self.slug, ref=ref) + actions = AggregatedCheckSuites.get(client=self.client, + owner=self.owner, + repo=self.slug, ref=ref) combined.status[actions.key] = actions except HTTPError as err: @@ -377,7 +376,8 @@ def get_build_status(self, revision: str, key: str) -> str: return status.state try: return self.get_commit_status(revision).status.get(key, None).state - except AttributeError: + except AttributeError as e: + LOG.error(e) return 'NOTSTARTED' def get_build_url(self, revision: str, key: str) -> str: @@ -508,17 +508,39 @@ def __str__(self) -> str: return self.state -class AggregatedCheckRuns(base.AbstractGitHostObject, - base.AbstractBuildStatus): +class WorkflowRun(base.AbstractGitHostObject): + """ + Endpoint to have access about workflows runs + """ + GET_URL = "/repos/{owner}/{repo}/actions/runs/{id}" + CREATE_URL = "/repos/{owner}/{repo}/actions/runs" + SCHEMA = schema.WorkflowRun + + +class AggregatedWorkflowRuns(base.AbstractGitHostObject): + GET_URL = "/repos/{owner}/{repo}/actions/runs" + SCHEMA = schema.AggregateWorkflowRuns + + @property + def total_count(self): + return self.data['total_count'] + + @property + def workflow_runs(self): + return self.data['workflow_runs'] + + +class AggregatedCheckSuites(base.AbstractGitHostObject, + base.AbstractBuildStatus): """ The Endpoint to have access infos about github actions runs """ - GET_URL = '/repos/{owner}/{repo}/commits/{ref}/check-runs' - SCHEMA = schema.AggregateCheckRuns + GET_URL = '/repos/{owner}/{repo}/commits/{ref}/check-suites' + SCHEMA = schema.AggregateCheckSuites def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._check_runs = [elem for elem in self.data['check_runs']] + self._check_suites = [elem for elem in self.data['check_suites']] @property def url(self): @@ -528,30 +550,79 @@ def sort_f(check_run): return datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ') return datetime(year=1962, month=1, day=1) - self.data['check_runs'].sort(key=sort_f, reverse=True) + self.data['check_suites'].sort(key=sort_f, reverse=True) - failed_c = [elem for elem in self.data['check_runs'] + failed_c = [elem for elem in self.data['check_suites'] if elem['conclusion'] != 'success'] if len(failed_c): return failed_c[0]['html_url'] - elif len(failed_c) and len(self.data['check_runs']): - return self.data['check_runs'][0]['html_url'] + elif len(failed_c) and len(self.data['check_suites']): + return self.data['check_suites'][0]['html_url'] return '' + def is_pending(self): + return len([ + elem for elem in self._check_suites if elem['status'] == 'pending' + ]) > 0 + + def _get_check_suite_ids(self, workflow_runs): + return list( + map(lambda elem: elem['check_suite_id'], workflow_runs) + ) + + def _get_aggregate_workflow_dispatched(self): + ref = self._check_suites[0]['head_sha'] + repo = self._check_suites[0]['repository']['name'] + owner = self._check_suites[0]['repository']['owner']['login'] + + return AggregatedWorkflowRuns.get( + client=self.client, + owner=owner, repo=repo, ref=ref, + params={ + 'event': 'workflow_dispatch', + 'branch': self._check_suites[0]['head_branch'] + }) + + def remove_unwanted_workflows(self): + """ + Remove two things: + - check-suites not triggerd by github-actions + - check-suites workflow triggerd by a `workflow_dispatch` event + """ + if self._check_suites.__len__() == 0: + return + + response = self._get_aggregate_workflow_dispatched() + dispatched = self._get_check_suite_ids(response.workflow_runs) + while len(dispatched) < response.total_count: + response = self._get_aggregate_workflow_dispatched() + dispatched += self._get_check_suite_ids(response.workflow_runs) + + self._check_suites = list(filter( + lambda elem: elem['id'] not in dispatched, + self._check_suites + )) + + self._check_suites = list(filter( + lambda elem: elem['app']['slug'] == 'github-actions', + self._check_suites + )) + @property def state(self): + self.remove_unwanted_workflows() all_complete = all( - elem['status'] == 'completed' for elem in self._check_runs + elem['status'] == 'completed' for elem in self._check_suites ) all_success = all( - elem['conclusion'] == 'success' or elem['conclusion'] == 'skipped' - for elem in self._check_runs + elem['conclusion'] == 'success' + for elem in self._check_suites ) - if self._check_runs.__len__() == 0: + if self._check_suites.__len__() == 0 or self.is_pending(): return 'NOTSTARTED' - elif self._check_runs.__len__() > 0 and not all_complete: + elif self._check_suites.__len__() > 0 and not all_complete: return 'INPROGRESS' - elif self._check_runs.__len__() > 0 and all_complete and all_success: + elif self._check_suites.__len__() > 0 and all_complete and all_success: return 'SUCCESSFUL' else: return 'FAILED' @@ -566,8 +637,8 @@ def key(self) -> str: @property def commit(self): - if self._check_runs.__len__() > 0: - return self._check_runs[0]["head_sha"] + if self._check_suites.__len__() > 0: + return self._check_suites[0]["head_sha"] else: return None diff --git a/bert_e/git_host/github/schema.py b/bert_e/git_host/github/schema.py index 2c5270bb..5a180051 100644 --- a/bert_e/git_host/github/schema.py +++ b/bert_e/git_host/github/schema.py @@ -78,11 +78,22 @@ class Branch(Schema): repo = fields.Nested(Repo) +class App(Schema): + id = fields.Int() + slug = fields.Str() + owner = fields.Nested(User) + name = fields.Str() + description = fields.Str() + + class CheckSuite(Schema): id = fields.Integer() head_sha = fields.Str() + head_branch = fields.Str() status = fields.Str() conclusion = fields.Str(allow_none=True) + repository = fields.Nested(Repo) + app = fields.Nested(App) class AggregateCheckSuites(Schema): @@ -98,6 +109,18 @@ class CheckRun(Schema): html_url = fields.Url() +class WorkflowRun(Schema): + id = fields.Integer() + head_sha = fields.Str() + head_branch = fields.Str() + status = fields.Str() + + +class AggregateWorkflowRuns(Schema): + total_count = fields.Integer() + workflow_runs = fields.Nested(WorkflowRun, many=True) + + class AggregateCheckRuns(Schema): total_count = fields.Integer() check_runs = fields.Nested(CheckRun, many=True) diff --git a/eve/main.yml b/eve/main.yml deleted file mode 100644 index 19f23070..00000000 --- a/eve/main.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- -version: 0.1 - -branches: - user/*, feature/*, improvement/*, bugfix/*, w/*, q/*, dependabot/*: - stage: pre-merge - -stages: - pre-merge: - worker: - type: local - steps: - - TriggerStages: - name: Syntax checker - haltOnFailure: true - stage_names: - - lint - - TriggerStages: - name: trigger all the tests - haltOnFailure: true - stage_names: - - tests - - tests-no-queue - - tests-queue - lint: - worker: - type: kube_pod - path: eve/worker/worker.yaml - images: - worker: eve/worker/dockerfile - steps: - - Git: &git_pull - name: git pull - repourl: "%(prop:git_reference)s" - shallow: True - retryFetch: True - haltOnFailure: True - - ShellCommand: - name: 'flake8' - haltOnFailure: true - command: tox -e flake8 - - ShellCommand: - name: 'check helm packaging' - command: helm lint charts/bert-e - haltOnFailure: true - - tests: - worker: &test_worker - type: kube_pod - path: eve/worker/worker.yaml - images: - worker: eve/worker/dockerfile - steps: - - Git: *git_pull - - ShellCommand: - name: unit tests - command: tox -e utests - - ShellCommand: - name: Git host API mock tests - command: tox -e tests-api-mock - - ShellCommand: - name: Bert-E server tests - command: tox -e tests-server - - ShellCommand: &coverage_report - name: Coverage report - command: tox -e coverage-report - tests-no-queue: - worker: *test_worker - steps: - - Git: *git_pull - - ShellCommand: - name: Bert-E tests (no queue support) - command: tox -e tests-noqueue - tests-queue: - worker: *test_worker - steps: - - Git: *git_pull - - ShellCommand: - name: Bert-E tests (with queue support) - command: tox -e tests - - tests-bitbucket: - worker: *test_worker - steps: - - Git: *git_pull - - ShellCommand: - name: Bert-E tests with real git host - command: tox -e tests-githost - env: - BERT_E_GITHOST: 'bitbucket' - BERT_E_REPO_PREFIX: '%(prop:bootstrap)s-%(prop:buildnumber)s-bert-e' - BERT_E_GITHOST_TEAM: '%(secret:bitbucket_team)s' - BERT_E_ROBOT_USERNAME: '%(secret:bitbucket_robot_username)s' - BERT_E_ROBOT_PASSWORD: '%(secret:bitbucket_robot_password)s' - BERT_E_CONTRIBUTOR_USERNAME: '%(secret:bitbucket_contributor_username)s' - BERT_E_CONTRIBUTOR_PASSWORD: '%(secret:bitbucket_contributor_password)s' - BERT_E_ADMIN_USERNAME: '%(secret:bitbucket_admin_username)s' - BERT_E_ADMIN_PASSWORD: '%(secret:bitbucket_admin_password)s' - - ShellCommand: *coverage_report diff --git a/eve/worker/dockerfile/Dockerfile b/eve/worker/dockerfile/Dockerfile deleted file mode 100644 index 9c094ebf..00000000 --- a/eve/worker/dockerfile/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -FROM ubuntu:bionic - -ENV HOME_BUILDBOT /var/lib/buildbot -ENV PYTHON_PIP_VERSION 21.0.1 -# Set the desired version of Helm -ENV DESIRED_VERSION v3.5.4 - -# -# Install packages needed by the buildchain -# - -RUN apt-get --assume-yes update \ - && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes \ - build-essential \ - ca-certificates \ - curl \ - git \ - libssl-dev \ - openssh-client \ - python \ - python3 \ - python3-dev \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python-dev \ - python-pip \ - python-pkg-resources \ - python-setuptools \ - sudo \ - tox \ - wget - -COPY requirements.txt /tmp/ - -RUN pip3 install pip==$PYTHON_PIP_VERSION -RUN pip3 install -r /tmp/requirements.txt - -# Install helm -RUN curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > /tmp/get_helm.sh && bash /tmp/get_helm.sh - -RUN git config --global credential.helper store - -# -# Add user eve -# - -RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \ - && adduser eve sudo \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# -# Eve configuration -# - -USER eve - -RUN mkdir -p /home/eve/workspace \ - && mkdir -p /home/eve/.ssh/ \ - && /bin/echo -e "Host bitbucket.org\n\tStrictHostKeyChecking no\n" >> /home/eve/.ssh/config - -ENV LANG=C.UTF-8 \ - BUILD_DISTRO=trusty - -# -# Run buildbot-worker on startup -# - -ARG BUILDBOT_VERSION -RUN sudo pip2 install buildbot-worker==$BUILDBOT_VERSION - -WORKDIR /home/eve/workspace -CMD buildbot-worker create-worker . "$BUILDMASTER:$BUILDMASTER_PORT" "$WORKERNAME" "$WORKERPASS" \ - && buildbot-worker start --nodaemon diff --git a/eve/worker/dockerfile/requirements.txt b/eve/worker/dockerfile/requirements.txt deleted file mode 100644 index 67740748..00000000 --- a/eve/worker/dockerfile/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -Authlib==0.8 -flask-session==0.3.1 -loginpass==0.1.1 -coverage==5.0a1 -flake8==3.5.0 -Flask==1.0.2 -Flask-Markdown==0.3 -Flask-WTF==0.14.2 -Jinja2==2.11.3 -jira==2.0.0 -Markdown==2.6.11 -marshmallow==2.15.4 -pip==20.0.1 -py-gfm==0.1.4 -pytest==3.7.1 -PyYAML==3.13 -raven==6.9.0 -requests==2.19.1 -requests-mock==1.5.2 -WTForms==2.2.1 diff --git a/eve/worker/worker.yaml b/eve/worker/worker.yaml deleted file mode 100644 index 173e1f26..00000000 --- a/eve/worker/worker.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "bert-e-worker" -spec: - containers: - - name: worker-pep8 - image: {{ images['worker'] }} - resources: - requests: - cpu: "1" - memory: 1Gi - limits: - cpu: "2" - memory: 2Gi - securityContext: - capabilities: - drop: - - ALL - # user -> 1000 - runAsUSer: 1000 - command: ["/bin/sh"] - args: ["-c", "buildbot-worker create-worker . ${BUILDMASTER}:${BUILDMASTER_PORT} ${WORKERNAME} ${WORKERPASS} && buildbot-worker start --nodaemon"] diff --git a/requirements.txt b/requirements.txt index 9e537aef..a8347cc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ Jinja2==2.10 jira==2.0.0 Markdown==2.6.11 marshmallow==2.15.4 +MarkupSafe==2.0.1 py-gfm==0.1.4 pytest==3.7.1 PyYAML==3.13