From b90f17bd2f2404472964cbd7e1ef70715675107a Mon Sep 17 00:00:00 2001 From: Valerii Svydenko Date: Wed, 31 Jan 2024 14:57:34 +0200 Subject: [PATCH] fix: update rhel build (#813) * fix: update rhel build Signed-off-by: Valeriy Svydenko * update license Signed-off-by: Valeriy Svydenko --------- Signed-off-by: Valeriy Svydenko --- build/dockerfiles/content_sets_centos8.repo | 11 -- build/dockerfiles/content_sets_rhel8.repo | 13 ++ build/dockerfiles/rhel.Dockerfile | 16 +- build/dockerfiles/rhel.entrypoint.sh | 149 +++++++++++++++++- .../generate_devworkspace_templates.sh | 2 + 5 files changed, 174 insertions(+), 17 deletions(-) delete mode 100644 build/dockerfiles/content_sets_centos8.repo create mode 100644 build/dockerfiles/content_sets_rhel8.repo diff --git a/build/dockerfiles/content_sets_centos8.repo b/build/dockerfiles/content_sets_centos8.repo deleted file mode 100644 index 4d72fcc33..000000000 --- a/build/dockerfiles/content_sets_centos8.repo +++ /dev/null @@ -1,11 +0,0 @@ -[centos-8-for-appstream] -name=centos-8-appstream -baseurl=http://mirror.centos.org/centos-8/8/AppStream/$basearch/os -enabled=1 -gpgcheck=0 - -[centos-8-for-baseos] -name=centos-8-baseos -baseurl=http://mirror.centos.org/centos-8/8/BaseOS/$basearch/os -enabled=1 -gpgcheck=0 diff --git a/build/dockerfiles/content_sets_rhel8.repo b/build/dockerfiles/content_sets_rhel8.repo new file mode 100644 index 000000000..989cf03cb --- /dev/null +++ b/build/dockerfiles/content_sets_rhel8.repo @@ -0,0 +1,13 @@ +[rhel-8-for-appstream-rpms-pulp] +name=rhel-8-for-appstream-rpms-pulp +baseurl=http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/$basearch/appstream/os +enabled=0 +gpgcheck=0 +skip_if_unavailable=True + +[rhel-8-for-baseos-rpms-pulp] +name=rhel-8-for-baseos-rpms-pulp +baseurl=http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/$basearch/baseos/os +enabled=0 +gpgcheck=0 +skip_if_unavailable=True diff --git a/build/dockerfiles/rhel.Dockerfile b/build/dockerfiles/rhel.Dockerfile index bb659c211..e5df3b05a 100644 --- a/build/dockerfiles/rhel.Dockerfile +++ b/build/dockerfiles/rhel.Dockerfile @@ -1,5 +1,5 @@ # -# Copyright (c) 2018-2021 Red Hat, Inc. +# Copyright (c) 2018-2024 Red Hat, Inc. # This program and the accompanying materials are made # available under the terms of the Eclipse Public License 2.0 # which is available at https://www.eclipse.org/legal/epl-2.0/ @@ -12,8 +12,8 @@ # # Builder: check meta.yamls and create index.json -# https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/ubi8/nodejs-16-minimal -FROM registry.access.redhat.com/ubi8/nodejs-16-minimal:1-155 as builder +# https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/ubi8/nodejs-18-minimal +FROM registry.access.redhat.com/ubi8/nodejs-18-minimal:1-91 as builder USER 0 ################# @@ -23,6 +23,8 @@ USER 0 ARG BOOTSTRAP=false ENV BOOTSTRAP=${BOOTSTRAP} +ARG VERSION + # to get all the python deps pre-fetched so we can build in Brew: # 1. extract files in the container to your local filesystem # find v3 -type f -exec dos2unix {} \; @@ -50,8 +52,11 @@ RUN ./check_mandatory_fields.sh devfiles RUN ./index.sh > /build/devfiles/index.json RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt -RUN ./generate_devworkspace_templates.sh +RUN ./generate_devworkspace_templates.sh $VERSION +RUN ./update_devworkspace_templates.sh $VERSION +RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt RUN chmod -R g+rwX /build/devfiles +RUN chmod -R g+rwX /build/resources ################# # PHASE TWO: configure registry image @@ -77,6 +82,8 @@ RUN sed -i /etc/httpd/conf/httpd.conf \ -e "s,logs/error_log,/dev/stderr," \ -e "s,logs/access_log,/dev/stdout," \ -e "s,AllowOverride None,AllowOverride All," && \ + echo "ServerName localhost" >> /etc/httpd/conf/httpd.conf && \ + echo "LimitRequestFieldSize 32768" >> /etc/httpd/conf/httpd.conf && \ chmod a+rwX /etc/httpd/conf /run/httpd /etc/httpd/logs/ STOPSIGNAL SIGWINCH @@ -85,6 +92,7 @@ WORKDIR /var/www/html RUN mkdir -m 777 /var/www/html/devfiles COPY .htaccess README.md /var/www/html/ COPY --from=builder /build/devfiles /var/www/html/devfiles +COPY --from=builder /build/resources /var/www/html/resources COPY --from=builder /build/devfiles/index.json /var/www/html/index COPY ./images /var/www/html/images COPY ./build/dockerfiles/rhel.entrypoint.sh ./build/dockerfiles/entrypoint.sh /usr/local/bin/ diff --git a/build/dockerfiles/rhel.entrypoint.sh b/build/dockerfiles/rhel.entrypoint.sh index 76184a383..aa450d5aa 100755 --- a/build/dockerfiles/rhel.entrypoint.sh +++ b/build/dockerfiles/rhel.entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018-2021 Red Hat, Inc. +# Copyright (c) 2018-2024 Red Hat, Inc. # This program and the accompanying materials are made # available under the terms of the Eclipse Public License 2.0 # which is available at https://www.eclipse.org/legal/epl-2.0/ @@ -14,7 +14,152 @@ if ! whoami &> /dev/null; then fi fi -set -x +set -e + +REGISTRY=${CHE_DEVFILE_IMAGES_REGISTRY_URL} +ORGANIZATION=${CHE_DEVFILE_IMAGES_REGISTRY_ORGANIZATION} +TAG=${CHE_DEVFILE_IMAGES_REGISTRY_TAG} +PUBLIC_URL=${CHE_DEVFILE_REGISTRY_URL} +INTERNAL_URL=${CHE_DEVFILE_REGISTRY_INTERNAL_URL} + +DEFAULT_DEVFILES_DIR="/var/www/html/devfiles" +DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}" +INDEX_JSON="${DEVFILES_DIR}/index.json" + +# Regex used to break an image reference into groups: +# \1 - Whitespace and (optional) quotation preceding image reference +# \2 - Registry portion of image, e.g. (quay.io)/che-incubator/che-code:tag +# \3 - Organization portion of image, e.g. quay.io/(che-incubator)/che-code:tag +# \4 - Image name portion of image, e.g. quay.io/che-incubator/(che-code):tag +# \5 - Optional image digest identifier (empty for tags), e.g. quay.io/che-incubator/che-code(@sha256):digest +# \6 - Tag of image or digest, e.g. quay.io/che-incubator/che-code:(tag) +# \7 - Optional quotation following image reference +IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)(@sha256)?:([._a-zA-Z0-9-]*)("?)' + +# Extract and use env variables with image digest information. +# Env variable name format: +# RELATED_IMAGES_(Image_name)_(Image_label)_(Encoded_base32_image_tag) +# Where are: +# "Image_name" - image name. Not valid chars for env variable name replaced to '_'. +# "Image_label" - image target, for example 'devfile_registry_image'. +# "Encoded_base32_image_tag_" - original image tag encoded to base32, to avoid invalid for env name chars. base32 alphabet has only +# one invalid character for env name: '='. That's why it was replaced to '_'. +# INFO: "=" for base32 it is pad character. If encoded string contains this char(s), then it is always located at the end of the string. +# Env value it is image with digest to use. +# Example env variable: +# RELATED_IMAGE_che_rust_1_39_devfile_registry_image_G4XDCMZOGIFA____=quay.io/eclipse/che-rust-1.39@sha256:3d9f36e6b3ed99c7a9959ac9476778ef5019add15b7c0f0b5f27b55587db3def +if env | grep -q ".*devfile_registry_image.*"; then + declare -A imageMap + readarray -t ENV_IMAGES < <(env | grep ".*devfile_registry_image.*") + for imageEnv in "${ENV_IMAGES[@]}"; do + tagOrDigest=$(echo "${imageEnv}" | sed -e 's;.*registry_image_\(.*\)=.*;\1;' | tr _ = | base32 -d) + if [[ ${tagOrDigest} == *"@"* ]]; then + # Well, image was "freezed", because it already has got digest, so do nothing. + continue + fi + imageWithDigest=${imageEnv#*=}; + if [[ -n "${tagOrDigest}" ]]; then + imageToReplace="${imageWithDigest%@*}:${tagOrDigest}" + else + imageToReplace="${imageWithDigest%@*}" + fi + digest="@${imageWithDigest#*@}" + imageMap["${imageToReplace}"]="${digest}" + done + + echo "--------------------------Digest map--------------------------" + for KEY in "${!imageMap[@]}"; do + echo "Key: $KEY Value: ${imageMap[${KEY}]}" + done + echo "--------------------------------------------------------------" + + readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-*.yaml') + for devfile in "${devfiles[@]}"; do + # Put an actual image on the same line with 'image' field + # It allows to grep image with digest in the next step + sed -i -E ':a;N;$!ba;s|image: >-\n[ ]+|image: |g' "${devfile}" + + readarray -t images < <(grep "image:" "${devfile}" | sed -r "s;.*image:[[:space:]]*'?\"?([._:a-zA-Z0-9-]*/?[._a-zA-Z0-9-]*/[._a-zA-Z0-9-]*(@sha256)?:?[._a-zA-Z0-9-]*)'?\"?[[:space:]]*;\1;") + for image in "${images[@]}"; do + separators="${image//[^\/]}" + # Warning, keep in mind: image without registry name is it possible case. It's mean, that image comes from private registry, where is we have organization name, but no registry name... + digest="${imageMap[${image}]}" + + if [[ -z "${digest}" ]] && [ "${#separators}" == "1" ]; then + imageWithDefaultRegistry="docker.io/${image}" + digest="${imageMap[${imageWithDefaultRegistry}]}" + fi + + if [[ -n "${digest}" ]]; then + if [[ ${image} == *":"* ]]; then + imageWithoutTag="${image%:*}" + tag="${image#*:}" + else + imageWithoutTag=${image} + tag="" + fi + + REGEX="([[:space:]]*\"?'?)(${imageWithoutTag}):?(${tag})(\"?'?)" + sed -i -E "s|image:${REGEX}|image:\1\2${digest}\4|" "${devfile}" + fi + done + done +fi + +# We can't use the `-d` option for readarray because +# registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2 +# The below command will fail if any path contains whitespace +readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-*.yaml') +readarray -t metas < <(find "${DEVFILES_DIR}" -name 'meta.yaml') +readarray -t templates < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-*.yaml') +for devfile in "${devfiles[@]}"; do + echo "Checking devfile $devfile" + # Need to update each field separately in case they are not defined. + # Defaults don't work because registry and tags may be different. + if [ -n "$REGISTRY" ]; then + echo " Updating image registry to $REGISTRY" + sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4\5:\6\7|" "$devfile" + fi + if [ -n "$ORGANIZATION" ]; then + echo " Updating image organization to $ORGANIZATION" + sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4\5:\6\7|" "$devfile" + fi + if [ -n "$TAG" ]; then + echo " Updating image tag to $TAG" + sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\7|" "$devfile" + fi +done + +if [ -n "$INTERNAL_URL" ]; then + INTERNAL_URL=${INTERNAL_URL%/} + echo "Updating internal URL in files to ${INTERNAL_URL}" + sed -i "s|{{ INTERNAL_URL }}|${INTERNAL_URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON" +fi + +if [ -n "$PUBLIC_URL" ]; then + echo "Updating devfiles to point at internal project zip files" + PUBLIC_URL=${PUBLIC_URL%/} + sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON" + + # Add PUBLIC_URL at the begining of 'icon' field and links ('self', 'che-incubator/che-code/latest') + sed -i "s|\"icon\": \"/images/|\"icon\": \"${PUBLIC_URL}/images/|" "$INDEX_JSON" + sed -i "s|\"self\": \"/devfiles/|\"self\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON" + sed -i "s|\"che-incubator/che-code/insiders\": \"/devfiles/|\"che-incubator/che-code/insiders\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON" + sed -i "s|\"che-incubator/che-code/latest\": \"/devfiles/|\"che-incubator/che-code/latest\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON" + sed -i "s|\"che-incubator/che-idea/next\": \"/devfiles/|\"che-incubator/che-idea/next\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON" +else + if grep -q '{{ DEVFILE_REGISTRY_URL }}' "${devfiles[@]}"; then + echo "WARNING: environment variable 'CHE_DEVFILE_REGISTRY_URL' not configured" \ + "This may cause issues with importing projects in a workspace." + # Experimental workaround -- detect service IP for che-devfile-registry + # Depends on service used being named 'che-devfile-registry' and only works + # within the cluster (i.e. browser-side retrieval won't work) + SERVICE_HOST=$(env | grep DEVFILE_REGISTRY_SERVICE_HOST= | cut -d '=' -f 2) + SERVICE_PORT=$(env | grep DEVFILE_REGISTRY_SERVICE_PORT= | cut -d '=' -f 2) + URL="http://${SERVICE_HOST}:${SERVICE_PORT}" + sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON" + fi +fi # start httpd if [[ -x /usr/sbin/httpd ]]; then diff --git a/build/scripts/generate_devworkspace_templates.sh b/build/scripts/generate_devworkspace_templates.sh index eab3c4f31..0bee53c40 100755 --- a/build/scripts/generate_devworkspace_templates.sh +++ b/build/scripts/generate_devworkspace_templates.sh @@ -10,6 +10,8 @@ set -e +npm install -g npm@10.4.0 + VERSION="${1%/}" if [[ -z "$VERSION" || "$VERSION" == *"-next" ]]; then VERSION="main"