add PG exts (#17)

* add exts * debug gpdb * update sql
QPod · Apr 12, 2024 · 90320a8 · 90320a8
1 parent 29dc39c
commit 90320a8
Show file tree

Hide file tree

Showing 11 changed files with 306 additions and 44 deletions.
diff --git a/docker_bigdata/Dockerfile b/docker_bigdata/Dockerfile
@@ -22,5 +22,4 @@ RUN source /opt/utils/script-setup.sh \
  && echo "Install redis-cli:" && setup_redis_client \
  && echo "Install pyspark:" && install_pip /opt/utils/list_install_pip_pyspark.txt \
  && echo "Install pyflink:" && install_pip /opt/utils/list_install_pip_pyflink.txt \
- && pip install --no-deps apache-flink \
  && echo "Clean up" && list_installed_packages && install__clean
diff --git a/docker_bigdata/work/list_install_pip_pyflink.txt b/docker_bigdata/work/list_install_pip_pyflink.txt
@@ -1,9 +1,2 @@
 % from: https://github.com/apache/flink/blob/master/flink-python/setup.py
-pemja
-pandas
-pyarrow
-apache-beam
-cloudpickle
-avro-python3
-requests
-% apache-flink % tempfix bu install without deps
+pemja pandas pyarrow apache-beam cloudpickle avro-python3 requests apache-flink
diff --git a/docker_bigdata/work/list_install_pip_pyspark.txt b/docker_bigdata/work/list_install_pip_pyspark.txt
@@ -1,4 +1,2 @@
 % from: https://github.com/apache/spark/blob/master/python/setup.py#L262
-pyspark
-pandas
-pyarrow
+pyspark pandas pyarrow
diff --git a/docker_greenplum/Dockerfile b/docker_greenplum/Dockerfile
@@ -8,10 +8,10 @@ COPY rootfs /
 RUN set -x && source /opt/utils/script-utils.sh \
  && install_apt /opt/utils/install_list_greenplum.apt \
  && apt-get -qq install -yq --no-install-recommends gcc g++ bison flex cmake pkg-config ccache ninja-build \
- && VERSION_GPDB_RELEASE=$(curl -sL https://github.com/greenplum-db/gpdb/releases.atom | grep 'releases/tag' | grep "7." | head -1 | grep -Po '\d[\d.]+' ) \
- && URL_GBDP="https://github.com/greenplum-db/gpdb/releases/download/${VERSION_GPDB_RELEASE}/${VERSION_GPDB_RELEASE}-src-full.tar.gz" \
+ && VERSION_GPDB_RELEASE=$(curl -sL https://github.com/greenplum-db/gpdb/releases.atom | grep 'releases/tag' | grep "/7." | head -1 | grep -Po '\d[\d.]+' ) \
+ && URL_GBDP="https://github.com/greenplum-db/gpdb/archive/refs/tags/${VERSION_GPDB_RELEASE}.tar.gz" \
  && echo "Downloading GBDP src release ${VERSION_GPDB_RELEASE} from: ${URL_GBDP}" \
- && install_tar_gz $URL_GBDP \
+ && install_tar_gz $URL_GBDP && mv /opt/gpdb-* /opt/gpdb_src \
  && cd /opt/gpdb_src \
  && PYTHON=/opt/conda/bin/python3 ./configure --prefix=/opt/gpdb --with-perl --with-python --with-libxml --with-gssapi --with-openssl \
  && sudo make -j16 && sudo make install -j16

diff --git a/docker_postgres/README.md b/docker_postgres/README.md
diff --git a/docker_postgres/postgres-ext.Dockerfile b/docker_postgres/postgres-ext.Dockerfile
@@ -6,26 +6,17 @@ FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
 
 LABEL maintainer="haobibo@gmail.com"
 
-COPY work /opt/utils/
+COPY rootfs /
 
-RUN set -x && . /opt/utils/script-utils.sh \
+RUN set -x && . /opt/utils/script-utils.sh && . /opt/utils/script-setup-pg_ext_mirror.sh \
  && apt-get update && apt-get install -y gettext \
- && DISTRO_NAME=$(awk '{ print tolower($0) }' <<< $(lsb_release -is)) \
- && DISTRO_CODE_NAME=$(lsb_release -cs) \
- # apt source for: https://packagecloud.io/citusdata/community
- && curl -fsSL "https://packagecloud.io/citusdata/community/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
- && echo "deb https://packagecloud.io/citusdata/community/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/citusdata_community.list \
- # apt source for: https://packagecloud.io/timescale/timescaledb
- && curl -fsSL "https://packagecloud.io/timescale/timescaledb/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/timescale_timescaledb.gpg \
- && echo "deb https://packagecloud.io/timescale/timescaledb/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/timescale_timescaledb.list \
- # apt source for: https://packagecloud.io/pigsty/pgsql
- && curl -fsSL "https://packagecloud.io/pigsty/pgsql/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/pigsty_pgsql.gpg \
- && echo "deb https://packagecloud.io/pigsty/pgsql/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/pigsty_pgsql.list \
  && envsubst < /opt/utils/install_list_pgext.tpl.apt > /opt/utils/install_list_pgext.apt \
  && rm -rf /opt/utils/install_list_pgext.tpl.apt \
  && echo "To install PG extensions: $(cat /opt/utils/install_list_pgext.apt)" \
  && install_apt /opt/utils/install_list_pgext.apt \
+ && . /opt/utils/script-setup-pg_ext.sh \
  && ls -alh /usr/share/postgresql/*/extension/*.control | sort \
+ && echo "include_dir='./conf.d'" >> /var/lib/postgresql/data/postgresql.conf \
  && echo "Clean up" && list_installed_packages && install__clean
 
 USER postgres

diff --git a/docker_postgres/rootfs/etc/security/limitd.d/80-postgres.conf b/docker_postgres/rootfs/etc/security/limitd.d/80-postgres.conf
@@ -0,0 +1,10 @@
+# ref: https://github.com/digoal/postgresql_docker_builder/blob/main/pg14_amd64/4.sh
+
+* soft nofile 1024000
+* hard nofile 1024000
+* soft nproc unlimited
+* hard nproc unlimited
+* soft core unlimited
+* hard core unlimited
+* soft memlock unlimited
+* hard memlock unlimited
diff --git a/docker_postgres/rootfs/opt/utils/install_list_pgext.tpl.apt b/docker_postgres/rootfs/opt/utils/install_list_pgext.tpl.apt
@@ -0,0 +1,83 @@
+postgresql-contrib
+postgresql-${PG_MAJOR}-postgis*
+postgresql-${PG_MAJOR}-pgvector
+postgresql-${PG_MAJOR}-cron
+postgresql-${PG_MAJOR}-wal2json
+timescaledb-2-postgresql-${PG_MAJOR}
+postgresql-${PG_MAJOR}-citus-12.1
+pg-graphql % https://github.com/supabase/pg_graphql
+pg-net % https://github.com/supabase/pg_net
+postgresql-${PG_MAJOR}-dirtyread
+postgresql-${PG_MAJOR}-extra-window-functions
+postgresql-${PG_MAJOR}-first-last-agg
+postgresql-${PG_MAJOR}-hll
+postgresql-${PG_MAJOR}-hypopg
+postgresql-${PG_MAJOR}-ip4r
+postgresql-${PG_MAJOR}-jsquery
+postgresql-${PG_MAJOR}-pgmemcache
+postgresql-${PG_MAJOR}-pljava
+postgresql-${PG_MAJOR}-pllua
+postgresql-${PG_MAJOR}-plpgsql-check
+postgresql-${PG_MAJOR}-plproxy
+postgresql-${PG_MAJOR}-prefix
+postgresql-${PG_MAJOR}-rational
+postgresql-${PG_MAJOR}-rdkit
+postgresql-${PG_MAJOR}-pg-qualstats
+postgresql-${PG_MAJOR}-pg-stat-kcache
+postgresql-${PG_MAJOR}-pg-wait-sampling
+postgresql-${PG_MAJOR}-pgfincore
+postgresql-${PG_MAJOR}-pgaudit
+postgresql-${PG_MAJOR}-pgpool2
+postgresql-${PG_MAJOR}-pgrouting
+postgresql-${PG_MAJOR}-pgrouting-doc
+postgresql-${PG_MAJOR}-pgrouting-scripts
+postgresql-${PG_MAJOR}-pgsphere
+postgresql-${PG_MAJOR}-pldebugger
+postgresql-${PG_MAJOR}-pointcloud
+postgresql-${PG_MAJOR}-plr
+postgresql-${PG_MAJOR}-powa
+postgresql-${PG_MAJOR}-q3c
+postgresql-${PG_MAJOR}-repack
+postgresql-${PG_MAJOR}-rum
+postgresql-${PG_MAJOR}-show-plans
+postgresql-${PG_MAJOR}-similarity
+postgresql-${PG_MAJOR}-tablelog
+postgresql-${PG_MAJOR}-tdigest
+postgresql-${PG_MAJOR}-plprofiler
+postgresql-${PG_MAJOR}-partman
+postgresql-${PG_MAJOR}-icu-ext
+postgresql-${PG_MAJOR}-orafce
+postgresql-${PG_MAJOR}-mysql-fdw
+postgresql-${PG_MAJOR}-oracle-fdw
+postgresql-${PG_MAJOR}-ogr-fdw
+postgresql-${PG_MAJOR}-tds-fdw
+postgresql-${PG_MAJOR}-credcheck
+postgresql-${PG_MAJOR}-decoderbufs
+postgresql-${PG_MAJOR}-mimeo
+postgresql-${PG_MAJOR}-pgmp
+postgresql-${PG_MAJOR}-preprepare
+postgresql-${PG_MAJOR}-prioritize
+postgresql-${PG_MAJOR}-squeeze
+postgresql-${PG_MAJOR}-toastinfo
+postgresql-${PG_MAJOR}-unit
+postgresql-${PG_MAJOR}-auto-failover
+pg-auto-failover-cli
+postgresql-plpython3-${PG_MAJOR}
+pgagroal
+pgpool2
+pgbouncer
+pgxnclient
+pgagent
+pgreplay
+pgbackrest
+pgbackrest-doc
+powa-collector
+elephant-shed-pgbackrest
+libpq-dev
+pgbadger
+pgloader
+ora2pg
+
+% postgresql-*-pgdg-pgroonga % TODO: postgresql-15-pgdg-pgroonga : Depends: libgroonga0 (>= 13.0.8) but 13.0.0+dfsg-3~deb12u1 is to be installed 
+% postgresql-pgml-* % TODO: many python packages
+% echo "deb https://apt.postgresml.org ${DISTRO_CODE_NAME} maintainer" | sudo tee /etc/apt/sources.list.d/postgresml.list
diff --git a/docker_postgres/rootfs/opt/utils/script-setup-pg_ext.sh b/docker_postgres/rootfs/opt/utils/script-setup-pg_ext.sh
@@ -0,0 +1,11 @@
+export USE_PGXS=1
+
+setup_apache_age() {
+ cd /tmp
+ git clone --depth 1 -b PG15 https://github.com/apache/age
+ cd /tmp/age
+ make -j8 && make install
+}
+
+
+# setup_apache_age
diff --git a/docker_postgres/rootfs/opt/utils/script-setup-pg_ext_mirror.sh b/docker_postgres/rootfs/opt/utils/script-setup-pg_ext_mirror.sh
@@ -0,0 +1,14 @@
+export DISTRO_NAME=$(awk '{ print tolower($0) }' <<< $(lsb_release -is))
+export DISTRO_CODE_NAME=$(lsb_release -cs)
+
+# apt source for: https://packagecloud.io/citusdata/community
+curl -fsSL "https://packagecloud.io/citusdata/community/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg
+echo "deb https://packagecloud.io/citusdata/community/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/citusdata_community.list
+
+# apt source for: https://packagecloud.io/timescale/timescaledb
+curl -fsSL "https://packagecloud.io/timescale/timescaledb/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/timescale_timescaledb.gpg
+echo "deb https://packagecloud.io/timescale/timescaledb/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/timescale_timescaledb.list
+
+# apt source for: https://packagecloud.io/pigsty/pgsql
+curl -fsSL "https://packagecloud.io/pigsty/pgsql/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/pigsty_pgsql.gpg
+echo "deb https://packagecloud.io/pigsty/pgsql/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/pigsty_pgsql.list
diff --git a/docker_postgres/work/install_list_pgext.tpl.apt b/docker_postgres/work/install_list_pgext.tpl.apt