From c75e07fa6c254244b29358b5d8004f4659f91eb8 Mon Sep 17 00:00:00 2001 From: Jon Leech <4693344+oddhack@users.noreply.github.com> Date: Tue, 9 Jul 2024 08:54:27 -0700 Subject: [PATCH] Use Khronos asciidoctor-spec Docker image in CI (#1196) * Use Khronos asciidoctor-spec Docker image in CI Per discussion with $bashbaug N.b. at present the CI script has less parallelism than it could, at least as I understand Actions. Some of the 'steps' could be split off into 'jobs'. Might try that next once the basic build is working. Net performance is still somewhat faster than current CI since it's generally faster to load the container than to add needed packages at each invocation, and the spec build is pretty fast, so there's not much to be gained. There was odd error behavior from shifting to the container which I have never seen in Vulkan CI, having to do with mixed ownership of files in the checked-out repository. I inserted a brute-force workaround right after the checkout action. * Update image (SHA changed, though not contents) * Update to 20240702 Docker image which sets HOME=/tmp to avoid asciidoctor-pdf permission problems with tmpfiles in home directory / * Fix SHA and remove fixed parallel job limit on manhtmlpages build * Switch to '#!/usr/bin/env python3' shebang lines Since the Docker build image runs a python virtual environment now. Also added 'scripts/runDocker' script which will invoke docker locally with the same image used in Github CI, for testing. Note this script will pull over a GB of Docker stuff onto the machine it's invoked on, if the image is not already cached. * Empty commit to try and re-trigger the 'fatal' message... ... which appears sporadic, not easily replicable. * Try to bulletproof the git invocations in Makefile * Remove diagnostic job stage after bulletproofing (hopefully) the Makefile For future reference, some of the git operations in CI and the Makefile appear to *sporadically* fail in CI because of different checked-out repo configurations. I modified the 'git symbolic-ref' and 'git log' operations invoked from the Makefile to detect errors and substitute a placeholder message, based on similar changes to the Vulkan Makefile a while back. This (appears) to eliminate the sporadic 'fatal' messages. We may need to do that to the 'git describe' as well. None of this reads on the generated artifacts, except that they may or may not contain accurate tag / commit comments. --- .github/workflows/presubmit.yml | 55 ++++++++++++++++------------ Makefile | 6 ++- scripts/apiconventions.py | 2 +- scripts/cgenerator.py | 2 +- scripts/checklinks.py | 2 +- scripts/clconventions.py | 2 +- scripts/docgenerator.py | 2 +- scripts/extensionmetadocgenerator.py | 2 +- scripts/genRef.py | 2 +- scripts/gen_dictionaries.py | 2 +- scripts/gen_version_notes.py | 2 +- scripts/gencl.py | 2 +- scripts/generator.py | 2 +- scripts/parse_dependency.py | 2 +- scripts/pygenerator.py | 2 +- scripts/realign.py | 2 +- scripts/reflib.py | 2 +- scripts/reg.py | 2 +- scripts/runDocker | 30 +++++++++++++++ scripts/scriptgenerator.py | 2 +- scripts/spec_tools/conventions.py | 2 +- 21 files changed, 83 insertions(+), 44 deletions(-) create mode 100755 scripts/runDocker diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 60557125..6a2d54b3 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -3,47 +3,54 @@ name: Presubmit permissions: contents: read -on: [push, pull_request] +# Controls when the action will run. +on: + push: + workflow_dispatch: + pull_request: +# These jobs are potentially parallelizeable jobs: build: - name: Build all specs + name: Build spec artifacts runs-on: ubuntu-latest + # Refer to the build container by its SHA instead of the name, to + # prevent caching problems when updating the image. + # container: khronosgroup/docker-images:asciidoctor-spec.20240702 + container: khronosgroup/docker-images@sha256:4aab96a03ef292439c9bd0f972adfa29cdf838d0909b1cb4ec2a6d7b2d14a37f steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 submodules: recursive + # If fetch-depth: 0 is not specified, then + # git describe --tags --dirty + # below fails. + # This could also affect SPECREVISION in the Makefile. + fetch-depth: 0 - - name: Install required packages + # Ownerships in the working directory are odd. + # . is owned by UID 1001, while repo files are owned by root. + # This leads to many odd messages like + # fatal: detected dubious ownership in repository at '/__w/OpenCL-Docs/OpenCL-Docs' + # The 'git config' is a brute-force workaround. + - name: Git safe directory workaround run: | - sudo apt-get install -y libpango1.0-dev libwebp-dev ghostscript fonts-lyx jing libavalon-framework-java libbatik-java python3-pyparsing - sudo gem install asciidoctor -v 2.0.16 - sudo gem install coderay -v 1.1.1 - sudo gem install rouge -v 3.19.0 - sudo gem install ttfunk -v 1.7.0 - sudo gem install hexapdf -v 0.27.0 - sudo gem install asciidoctor-pdf -v 2.3.4 - sudo gem install asciidoctor-mathematical -v 0.3.5 - sudo pip install pyparsing - - - name: List git tag + git config --global --add safe.directory '*' + ls -lda . .. .git Makefile + + - name: Validate XML run: | - git describe --tags --dirty + make -C xml validate - name: Generate core specs (HTML and PDF) run: | - python3 makeSpec -clean -spec core OUTDIR=out.core -j 5 api c env ext cxx4opencl - + python3 makeSpec -clean -spec core OUTDIR=out.core -j 5 -O api c env ext cxx4opencl + - name: Generate core + extension specs (HTML) run: | - python3 makeSpec -clean -spec khr OUTDIR=out.khr -j 12 html + python3 makeSpec -clean -spec khr OUTDIR=out.khr -j -O html - name: Generate reference pages run: | - python3 makeSpec -spec khr OUTDIR=out.refpages -j 12 manhtmlpages - - - name: Validate XML - run: | - make -C xml validate + python3 makeSpec -spec khr OUTDIR=out.refpages -j -O manhtmlpages diff --git a/Makefile b/Makefile index 1540b834..c2e00d07 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ EXTOPTIONS := $(foreach ext,$(EXTS),-extension $(ext)) QUIET ?= VERYQUIET ?= @ +PYTHON ?= python3 ASCIIDOCTOR ?= asciidoctor RM = rm -f RMRF = rm -rf @@ -72,8 +73,8 @@ SPECREVISION = $(shell echo `git describe --tags --dirty`) # This used to be a dependency in the spec html/pdf targets, # but that's likely to lead to merge conflicts. Just regenerate # when pushing a new spec for review to the sandbox. -SPECREMARK = from git branch: $(shell echo `git symbolic-ref --short HEAD`) \ - commit: $(shell echo `git log -1 --format="%H"`) +SPECREMARK = from git branch: $(shell echo `git symbolic-ref --short HEAD 2> /dev/null || echo Git branch not available`) \ + commit: $(shell echo `git log -1 --format="%H" 2> /dev/null || echo Git commit not available`) endif # The C++ for OpenCL document revision scheme is aligned with its release date. # Revision naming scheme is as follows: @@ -545,6 +546,7 @@ $(METADEPEND): $(APIXML) $(GENSCRIPT) attribs: $(ATTRIBFILE) $(ATTRIBFILE): + $(QUIET)$(MKDIR) $(dir $@) for attrib in $(EXTS) ; do \ echo ":$${attrib}:" ; \ done > $@ diff --git a/scripts/apiconventions.py b/scripts/apiconventions.py index 4d27d04f..f16dcd47 100644 --- a/scripts/apiconventions.py +++ b/scripts/apiconventions.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2021-2024 The Khronos Group Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/cgenerator.py b/scripts/cgenerator.py index f86658ee..713113c0 100644 --- a/scripts/cgenerator.py +++ b/scripts/cgenerator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/checklinks.py b/scripts/checklinks.py index 816f601a..5d79a676 100755 --- a/scripts/checklinks.py +++ b/scripts/checklinks.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # Copyright 2013-2024 The Khronos Group Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/clconventions.py b/scripts/clconventions.py index c561eb73..734041ae 100644 --- a/scripts/clconventions.py +++ b/scripts/clconventions.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/docgenerator.py b/scripts/docgenerator.py index b714ef7c..b8c2afd0 100644 --- a/scripts/docgenerator.py +++ b/scripts/docgenerator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/extensionmetadocgenerator.py b/scripts/extensionmetadocgenerator.py index a200bab9..9f93a29e 100644 --- a/scripts/extensionmetadocgenerator.py +++ b/scripts/extensionmetadocgenerator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/genRef.py b/scripts/genRef.py index 9b78fd0d..2eabd231 100755 --- a/scripts/genRef.py +++ b/scripts/genRef.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # Copyright 2016-2024 The Khronos Group Inc. # diff --git a/scripts/gen_dictionaries.py b/scripts/gen_dictionaries.py index 4f220aa1..e513385f 100755 --- a/scripts/gen_dictionaries.py +++ b/scripts/gen_dictionaries.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2019-2024 The Khronos Group Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/gen_version_notes.py b/scripts/gen_version_notes.py index 21271643..d5cd81ff 100755 --- a/scripts/gen_version_notes.py +++ b/scripts/gen_version_notes.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2019-2024 The Khronos Group Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/gencl.py b/scripts/gencl.py index 04f821ce..9cc8a1d2 100755 --- a/scripts/gencl.py +++ b/scripts/gencl.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/generator.py b/scripts/generator.py index dea2ffa3..c534faf8 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/parse_dependency.py b/scripts/parse_dependency.py index 5d204959..071d7b3c 100755 --- a/scripts/parse_dependency.py +++ b/scripts/parse_dependency.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2022-2024 The Khronos Group Inc. # Copyright 2003-2019 Paul McGuire diff --git a/scripts/pygenerator.py b/scripts/pygenerator.py index 6656b460..8656587e 100644 --- a/scripts/pygenerator.py +++ b/scripts/pygenerator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/realign.py b/scripts/realign.py index 495cb74b..71f9f85e 100755 --- a/scripts/realign.py +++ b/scripts/realign.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # Copyright 2013-2024 The Khronos Group Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/reflib.py b/scripts/reflib.py index 41fec492..bd873a36 100644 --- a/scripts/reflib.py +++ b/scripts/reflib.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # Copyright 2016-2024 The Khronos Group Inc. # diff --git a/scripts/reg.py b/scripts/reg.py index b8f8af7c..d5495212 100755 --- a/scripts/reg.py +++ b/scripts/reg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/runDocker b/scripts/runDocker new file mode 100755 index 00000000..19bf0058 --- /dev/null +++ b/scripts/runDocker @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2022-2024 The Khronos Group Inc. +# SPDX-License-Identifier: Apache-2.0 + +# runDocker - run the Khronos `asciidoctor-spec` Docker image with a local +# clone of the specification repository. +# The following command-line tools are required to run this script: +# awk dirname docker grep id realpath +# These are all normal Linux developer tools except for 'docker' itself. + +# Determine path to repository root directory +scriptpath=`dirname $0` +repopath=`realpath $scriptpath/..` + +# Get SHA256 of the asciidoctor-spec image build used by CI. +image=`grep -m 1 khronosgroup/docker-images@sha256: $repopath/.github/workflows/presubmit.yml | \ + awk '{print $2}'` + +uid=`id -u` +gid=`id -g` +echo "Executing Docker with spec build image and mounted spec repository root:" + +# --user causes Docker to run as the specified UID:GID instead of as root +# -it runs interactively and uses a pseudotty +# --rm removes the container on exit +# -v mounts the repository clone as /vulkan in the container +# $image is image to run +# /bin/bash drops into a shell in the container +set -x +docker run --network=host --user ${uid}:${gid} -it --rm -v ${repopath}:/opencl $image /bin/bash diff --git a/scripts/scriptgenerator.py b/scripts/scriptgenerator.py index f5ed14d0..27339b26 100644 --- a/scripts/scriptgenerator.py +++ b/scripts/scriptgenerator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. # diff --git a/scripts/spec_tools/conventions.py b/scripts/spec_tools/conventions.py index 50ca75d4..190bd3b8 100644 --- a/scripts/spec_tools/conventions.py +++ b/scripts/spec_tools/conventions.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -i +#!/usr/bin/env python3 -i # # Copyright 2013-2024 The Khronos Group Inc. #