From f3737b0b2f441e55a2e8a8e51a0af2fb0e66ee60 Mon Sep 17 00:00:00 2001 From: mrdrivingduck Date: Thu, 5 Sep 2024 13:53:50 +0800 Subject: [PATCH] ci: support packaging RPM and DEB --- .github/workflows/package.yml | 124 +++++++++++++++++++++ package/debian/build-deb.sh | 153 ++++++++++++++++++++++++++ package/debian/control | 17 +++ package/rpm/PolarDB.spec | 199 ++++++++++++++++++++++++++++++++++ package/rpm/build-rpm.sh | 39 +++++++ 5 files changed, 532 insertions(+) create mode 100644 .github/workflows/package.yml create mode 100755 package/debian/build-deb.sh create mode 100644 package/debian/control create mode 100644 package/rpm/PolarDB.spec create mode 100755 package/rpm/build-rpm.sh diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 00000000000..7157c8c9552 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,124 @@ +name: build packages + +on: + push: + tags: + - "v*" + branches: [ POLARDB_15_STABLE ] + pull_request: + branches: [ POLARDB_15_STABLE ] + paths: + - '.github/workflows/package.yml' + - 'config/**' + - 'contrib/**' + - 'external/**' + - 'package/**' + - 'src/**' + - 'configure*' + - 'build.sh' + # trigger testing manually + workflow_dispatch: + +jobs: + + build-and-publish-rpm: + runs-on: ubuntu-latest + strategy: + matrix: + base_image: [ anolis8, rocky8, rocky9 ] + steps: + - name: fetch source code + uses: actions/checkout@v4 + - name: create and start the container + run: | + docker create \ + -t \ + --name polardb_${{ matrix.base_image }} \ + -v `pwd`:/home/postgres/PolarDB-for-PostgreSQL \ + polardb/polardb_pg_devel:${{ matrix.base_image }} \ + bash && \ + docker start polardb_${{ matrix.base_image }} + + - name: build RPM package + run: | + docker exec \ + polardb_${{ matrix.base_image }} \ + bash -c "cd /home/postgres/PolarDB-for-PostgreSQL && \ + sudo chown -R postgres:postgres ./ && \ + cd ./package/rpm && \ + if [ -f /etc/bashrc ]; then source /etc/bashrc; fi && \ + ./build-rpm.sh" + + - name: get package name + run: | + cd ./package/rpm/x86_64 && RPMNAME=`ls` && echo "rpmname=$RPMNAME" >> $GITHUB_ENV + - name: upload artifact + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.rpmname }} + path: ./package/rpm/x86_64/${{ env.rpmname }} + compression-level: 0 + - name: change ownership of the source code + run: | + sudo chown -R runner:runner `pwd` + + build-and-publish-deb: + runs-on: ubuntu-latest + strategy: + matrix: + base_image: [ ubuntu20.04, ubuntu22.04, ubuntu24.04 ] + steps: + - name: fetch source code + uses: actions/checkout@v4 + - name: create and start the container + run: | + docker create \ + -t \ + --name polardb_${{ matrix.base_image }} \ + -v `pwd`:/home/postgres/PolarDB-for-PostgreSQL \ + polardb/polardb_pg_devel:${{ matrix.base_image }} \ + bash && \ + docker start polardb_${{ matrix.base_image }} + + - name: build deb package + run: | + docker exec \ + polardb_${{ matrix.base_image }} \ + bash -c "cd /home/postgres/PolarDB-for-PostgreSQL && \ + sudo chown -R postgres:postgres ./ && \ + cd ./package/debian && \ + if [ -f /etc/bashrc ]; then source /etc/bashrc; fi && \ + ./build-deb.sh" + + - name: get package name + run: | + cd ./package/debian && DEBNAME=`ls *.deb` && echo "debname=$DEBNAME" >> $GITHUB_ENV + - name: upload artifact + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v4 + with: + name: ${{ env.debname }} + path: ./package/debian/${{ env.debname }} + compression-level: 0 + - name: change ownership of the source code + run: | + sudo chown -R runner:runner `pwd` + + release-packages: + if: startsWith(github.ref, 'refs/tags/') + needs: [ build-and-publish-rpm, build-and-publish-deb ] + runs-on: ubuntu-latest + steps: + - name: download all artifacts + uses: actions/download-artifact@v4 + with: + pattern: | + *.{rpm,deb} + merge-multiple: true + - name: upload artifacts to release + uses: softprops/action-gh-release@v2 + with: + files: | + *.rpm + *.deb diff --git a/package/debian/build-deb.sh b/package/debian/build-deb.sh new file mode 100755 index 00000000000..d242b59b36c --- /dev/null +++ b/package/debian/build-deb.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# build-deb.sh +# Use shell script to build .deb packages. +# +# Copyright (c) 2024, Alibaba Group Holding Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# IDENTIFICATION +# package/debian/build-deb.sh + +build_date=${1:-$(date +"%Y%m%d%H%M%S")} +code_commit=$(git rev-parse HEAD || echo unknown) +code_branch=$(git rev-parse --abbrev-ref HEAD || echo unknown) +polar_commit=$(git rev-parse --short=8 HEAD || echo unknown) +pg_version=$(grep AC_INIT ../../configure.ac | awk -F'[][]' '{print $4}') +polar_majorversion=$(echo ${pg_version}) +polar_minorversion=$(grep '^POLAR_MINORVERSION' ../../configure.ac | cut -d '"' -f2) +polar_release_date=$(grep '^POLAR_RELEASE_DATE' ../../configure.ac | cut -d '"' -f2) +polar_version=${polar_majorversion}.${polar_minorversion} +polar_pg_version=${pg_version}.${polar_minorversion} + +package=PolarDB +debpkgname=$(grep 'Package:' ./control | awk '{print $2}') +distname=$(grep '^ID=' /etc/os-release | awk -F'=' '{print $2}') +distversion=$(grep 'VERSION_ID=' /etc/os-release | awk -F\" '{print $2}') +arch=$(dpkg --print-architecture) + +pkgname=${package}_${polar_version}-${polar_commit}-${distname}${distversion}_${arch} +rm -rf ./${pkgname} +mkdir -p ${pkgname}/DEBIAN + +cat ./control >> ${pkgname}/DEBIAN/control +echo 'Version: '${polar_version}'-'${polar_commit}'' >> ${pkgname}/DEBIAN/control +echo 'Architecture: '${arch}'' >> ${pkgname}/DEBIAN/control + +prefix=/u01/polardb_pg +buildroot=$(pwd) +cd ../../ +export COPT="-Wno-error ${COPT-}" +./build.sh --ec="--prefix=${buildroot}/${pkgname}${prefix} --with-pfsd" --debug=off --ni +cd ${buildroot} + +polar_install_dependency() +{ + target_dir=${1} + + # create link for lib inside lib + cd ${target_dir}/lib/ + ln -sf ../lib ./lib + + cd ${target_dir} + + # generate list of .so files + # collect all the executable binaries, scripts and shared libraries + binfiles=`find ${target_dir}/bin` + libfiles=`find ${target_dir}/lib` + filelist=${binfiles}$'\n'${libfiles} + exelist=`echo $filelist | xargs -r file | egrep -v ":.* (commands|script)" | \ + grep ":.*executable" | cut -d: -f1` + scriptlist=`echo $filelist | xargs -r file | \ + egrep ":.* (commands|script)" | cut -d: -f1` + liblist=`echo $filelist | xargs -r file | \ + grep ":.*shared object" | cut -d : -f1` + + # dependency list of the executable binaries and shared libraries + cp /dev/null mytmpfilelist + cp /dev/null mytmpfilelist2 + + # put PolarDB-PG libs before any other libs to let ldd take it first + export LD_LIBRARY_PATH=${target_dir}/lib:$LD_LIBRARY_PATH:/usr/lib + + # dependency list of all binaries and shared objects + for f in $liblist $exelist; do + ldd $f | awk '/=>/ { + if (X$3 != "X" && $3 !~ /libNoVersion.so/ && $3 !~ /4[um]lib.so/ && $3 !~ /libredhat-kernel.so/ && $3 !~ /libselinux.so/ && $3 !~ /\/u01\/polardb_pg/ && $3 !~ /libjvm.so/ && $3 ~ /\.so/) { + # printf "$s => $s\n", $1, $3 + print $3 + } + }' >> mytmpfilelist + done + + # deduplicate + cat mytmpfilelist | sort -u > mytmpfilelist2 + + for f in `cat mytmpfilelist2`; do + ldd $f | awk '/=>/ { + if (X$3 != "X" && $3 !~ /libNoVersion.so/ && $3 !~ /4[um]lib.so/ && $3 !~ /libredhat-kernel.so/ && $3 !~ /libselinux.so/ && $3 !~ /\/u01\/polardb_pg/ && $3 !~ /libjvm.so/ && $3 ~ /\.so/) { + # printf "$s => $s\n", $1, $3 + print $3 + } + }' >> mytmpfilelist + done + + # deduplicate + cat mytmpfilelist | sort -u > mytmpfilelist2 + + # copy libraries if necessary + for line in `cat mytmpfilelist2`; do + base=`basename $line` + dirpath=${target_dir}/lib + filepath=$dirpath/$base + + objdump -p $line | awk 'BEGIN { START=0; LIBNAME=""; } + /^$/ { START=0; } + /^Dynamic Section:$/ { START=1; } + (START==1) && /NEEDED/ { + print $2 ; + } + (START==2) && /^[A-Za-z]/ { START=3; } + /^Version References:$/ { START=2; } + (START==2) && /required from/ { + sub(/:/, "", $3); + LIBNAME=$3; + } + (START==2) && (LIBNAME!="") && ($4!="") && (($4~/^GLIBC_*/) || ($4~/^GCC_*/)) { + print LIBNAME "(" $4 ")"; + } + END { exit 0 } + ' > objdumpfile + + has_private= + if grep -q PRIVATE objdumpfile; then + has_private=true + fi + + if [[ ! -f $filepath ]]; then + if [[ $has_private != "true" ]]; then + cp $line $dirpath + echo $line $dirpath + fi + fi + done + + rm mytmpfilelist mytmpfilelist2 objdumpfile +} + +# install package dependencies +polar_install_dependency $(pwd)/${pkgname}${prefix} +cd ${buildroot} + +dpkg --build ./${pkgname} diff --git a/package/debian/control b/package/debian/control new file mode 100644 index 00000000000..c4309d327aa --- /dev/null +++ b/package/debian/control @@ -0,0 +1,17 @@ +Package: polardb-for-postgresql +Maintainer: mrdrivingduck +Depends: libicu66 | libicu70 | libicu74, libkrb5-3, libldap-2.4-2 | libldap-2.5-0 | libldap2 +Section: database +Priority: optional +Homepage: https://github.com/ApsaraDB/PolarDB-for-PostgreSQL +Description: PolarDB is an advanced Object-Relational database management system + (DBMS) that supports almost all SQL constructs (including + transactions, subselects and user-defined types and functions). The + PolarDB package includes the client programs and libraries that + you'll need to access a PolarDB DBMS server. These PolarDB + client programs are programs that directly manipulate the internal + structure of PolarDB databases on a PolarDB server. These client + programs can be located on the same machine with the PolarDB + server, or may be on a remote machine which accesses a PolarDB + server over a network connection. This package contains the command-line + utilities for managing PolarDB databases on a PolarDB server. diff --git a/package/rpm/PolarDB.spec b/package/rpm/PolarDB.spec new file mode 100644 index 00000000000..1f3300f336a --- /dev/null +++ b/package/rpm/PolarDB.spec @@ -0,0 +1,199 @@ +# PolarDB.spec +# Spec file for building RPM package for PolarDB-PG +# +# Copyright (c) 2024, Alibaba Group Holding Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# IDENTIFICATION +# PolarDB.spec + +# compile options + +%define build_date %(echo $RELEASE) +%define code_commit %(git rev-parse HEAD || echo unknown) +%define code_branch %(git rev-parse --abbrev-ref HEAD || echo unknown) +%define polar_commit %(git rev-parse --short=8 HEAD || echo unknown) +%define pg_version %(grep AC_INIT ../../configure.ac | awk -F'[][]' '{print $4}') +%define polar_majorversion %{pg_version} +%define polar_minorversion %(grep '^POLAR_MINORVERSION' ../../configure.ac | cut -d '"' -f2) +%define polar_release_date %(grep '^POLAR_RELEASE_DATE' ../../configure.ac | cut -d '"' -f2) +%define polar_version %{polar_majorversion}.%{polar_minorversion} +%define polar_pg_version %{pg_version}.%{polar_minorversion} +%define _build_id_links none +%define __spec_install_pre /bin/true + +Version: %{polar_version} +Release: %{polar_commit}%{?dist} +Summary: PolarDB +Name: PolarDB +Group: alibaba/application +License: Commercial +BuildArch: x86_64 aarch64 +Prefix: /u01/polardb_pg +Provides: PolarDB +AutoReqProv: none + +%define copy_dir /u01/polardb_pg_%{polar_release_date} + +# Requires: libicu + +%description +CodeBranch: %{code_branch} +CodeCommit: %{code_commit} +PolarCommit: %{polar_commit} +PolarVersion: %{polar_version} +PolarPGVersion: %{polar_pg_version} +PolarReleaseDate: %{polar_release_date} +PFSDVersion: %{pfsd_version} +PolarDB is an advanced Object-Relational database management system +(DBMS) that supports almost all SQL constructs (including +transactions, subselects and user-defined types and functions). The +PolarDB package includes the client programs and libraries that +you'll need to access a PolarDB DBMS server. These PolarDB +client programs are programs that directly manipulate the internal +structure of PolarDB databases on a PolarDB server. These client +programs can be located on the same machine with the PolarDB +server, or may be on a remote machine which accesses a PolarDB +server over a network connection. This package contains the command-line +utilities for managing PolarDB databases on a PolarDB server. + +%build +export CC=gcc CXX=g++ +export NM=gcc-nm AR=gcc-ar RANLIB=gcc-ranlib +export COPT="-Wno-error ${COPT-}" + +cd $OLDPWD/../../ +DESTDIR=%{buildroot} ./build.sh --ec="--prefix=%{prefix} --with-pfsd" --debug=off --ni + +%install +polar_install_dependency() +{ + target_dir=${1} + + # create link for lib inside lib + cd %{buildroot}${target_dir}/lib/ + ln -sf ../lib ./lib + + cd %{buildroot} + + # generate list of .so files + # collect all the executable binaries, scripts and shared libraries + binfiles=`find %{buildroot}${target_dir}/bin` + libfiles=`find %{buildroot}${target_dir}/lib` + filelist=${binfiles}$'\n'${libfiles} + exelist=`echo $filelist | xargs -r file | egrep -v ":.* (commands|script)" | \ + grep ":.*executable" | cut -d: -f1` + scriptlist=`echo $filelist | xargs -r file | \ + egrep ":.* (commands|script)" | cut -d: -f1` + liblist=`echo $filelist | xargs -r file | \ + grep ":.*shared object" | cut -d : -f1` + + # dependency list of the executable binaries and shared libraries + cp /dev/null mytmpfilelist + cp /dev/null mytmpfilelist2 + + # put PolarDB-PG libs before any other libs to let ldd take it first + eval PERL_PATH=`cd /usr/lib64/perl[5-9]/CORE/ ; pwd` + export LD_LIBRARY_PATH=%{buildroot}${target_dir}/lib:$PERL_PATH:$LD_LIBRARY_PATH:/usr/lib + + # dependency list of all binaries and shared objects + for f in $liblist $exelist; do + ldd $f | awk '/=>/ { + if (X$3 != "X" && $3 !~ /libNoVersion.so/ && $3 !~ /4[um]lib.so/ && $3 !~ /libredhat-kernel.so/ && $3 !~ /libselinux.so/ && $3 !~ /\/u01\/polardb_pg/ && $3 !~ /libjvm.so/ && $3 ~ /\.so/) { + # printf "$s => $s\n", $1, $3 + print $3 + } + }' >> mytmpfilelist + done + + # deduplicate + cat mytmpfilelist | sort -u > mytmpfilelist2 + + for f in `cat mytmpfilelist2`; do + ldd $f | awk '/=>/ { + if (X$3 != "X" && $3 !~ /libNoVersion.so/ && $3 !~ /4[um]lib.so/ && $3 !~ /libredhat-kernel.so/ && $3 !~ /libselinux.so/ && $3 !~ /\/u01\/polardb_pg/ && $3 !~ /libjvm.so/ && $3 ~ /\.so/) { + # printf "$s => $s\n", $1, $3 + print $3 + } + }' >> mytmpfilelist + done + + # deduplicate + cat mytmpfilelist | sort -u > mytmpfilelist2 + + # copy libraries if necessary + for line in `cat mytmpfilelist2`; do + base=`basename $line` + dirpath=%{buildroot}${target_dir}/lib + filepath=$dirpath/$base + + objdump -p $line | awk 'BEGIN { START=0; LIBNAME=""; } + /^$/ { START=0; } + /^Dynamic Section:$/ { START=1; } + (START==1) && /NEEDED/ { + print $2 ; + } + (START==2) && /^[A-Za-z]/ { START=3; } + /^Version References:$/ { START=2; } + (START==2) && /required from/ { + sub(/:/, "", $3); + LIBNAME=$3; + } + (START==2) && (LIBNAME!="") && ($4!="") && (($4~/^GLIBC_*/) || ($4~/^GCC_*/)) { + print LIBNAME "(" $4 ")"; + } + END { exit 0 } + ' > objdumpfile + + has_private= + if grep -q PRIVATE objdumpfile; then + has_private=true + fi + + if [[ ! -f $filepath ]]; then + if [[ $has_private != "true" ]]; then + cp $line $dirpath + fi + fi + done + + rm mytmpfilelist mytmpfilelist2 objdumpfile + + # avoid failure of call to /usr/lib/rpm/check-buildroot + export QA_SKIP_BUILD_ROOT=1 +} + +# install package dependencies +polar_install_dependency %{prefix} + +# copy external file +%post + +if [[ -f %{copy_dir}/bin/postgres ]]; +then + echo WARNING: PolarDB-PG files already exist, file copying will be skipped! +else + if [[ -d %{copy_dir} ]]; + then + cp -rf %{prefix}/* %{copy_dir} + else + cp -rf %{prefix} %{copy_dir} + fi +fi + +%files +%defattr(-,root,root) +%{prefix}/* + +%define __os_install_post %{nil} diff --git a/package/rpm/build-rpm.sh b/package/rpm/build-rpm.sh new file mode 100755 index 00000000000..5b85bbbbec6 --- /dev/null +++ b/package/rpm/build-rpm.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# build-rpm.sh +# Use shell script to build .rpm packages. +# +# Copyright (c) 2024, Alibaba Group Holding Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# IDENTIFICATION +# package/rpm/build-rpm.sh + +RPM_MACROS=$HOME/.rpmmacros +TOP_DIR=`pwd`/.self_rpm_create + +release=${1:-$(date +"%Y%m%d%H%M%S")} + +rm -f $RPM_MACROS +echo "%_topdir $TOP_DIR" > $RPM_MACROS +echo "%_release $release" >> $RPM_MACROS + +rm -rf $TOP_DIR +RELEASE=$release rpmbuild -bb --buildroot $TOP_DIR/BUILD PolarDB.spec;st=$? + +if [ "$st" != 0 ]; then + exit $st +fi + +cp -rf $TOP_DIR/RPMS/* .