Add devcontainer for development purposes (#7102)

This change adds a devcontainer configuration to the Citus project. This
devcontainer allows for quick generation of isolated development
environments, either local on the machine of a developer or in a cloud,
like github codepaces.

The devcontainer is updated automatically by github actions when its
configuration changes.

For more detailed instructions on how to quickstart the development in a
container see CONTRIBUTING.md

.devcontainer/.gdbinit

@@ -0,0 +1,5 @@
+# gdbpg.py contains scripts to nicely print the postgres datastructures
+# while in a gdb session. Since the vscode debugger is based on gdb this
+# actually also works when debugging with vscode. Providing nice tools
+# to understand the internal datastructures we are working with.
+source /root/gdbpg.py

.devcontainer/.gitignore

@@ -0,0 +1 @@
+postgresql-*.tar.bz2

.devcontainer/.psqlrc

@@ -0,0 +1,7 @@
+\timing on
+\pset linestyle unicode
+\pset border 2
+\setenv PAGER 'pspg --no-mouse -bX --no-commandbar --no-topbar'
+\set HISTSIZE 100000
+\set PROMPT1 '\n%[%033[1m%]%M %n@%/:%>-%p%R%[%033[0m%]%# '
+\set PROMPT2 ' '

.devcontainer/.vscode/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+docopt = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

.devcontainer/.vscode/generate_c_cpp_properties-json.py

@@ -0,0 +1,84 @@
+#! /usr/bin/env pipenv-shebang
+"""Generate C/C++ properties file for VSCode.
+
+Uses pgenv to iterate postgres versions and generate
+a C/C++ properties file for VSCode containing the
+include paths for the postgres headers.
+
+Usage:
+ generate_c_cpp_properties-json.py <target_path>
+ generate_c_cpp_properties-json.py (-h | --help)
+ generate_c_cpp_properties-json.py --version
+
+Options:
+ -h --help Show this screen.
+ --version Show version.
+
+"""
+import json
+import subprocess
+
+from docopt import docopt
+
+
+def main(args):
+ target_path = args['<target_path>']
+
+ output = subprocess.check_output(['pgenv', 'versions'])
+ # typical output is:
+ # 14.8 pgsql-14.8
+ # * 15.3 pgsql-15.3
+ # 16beta2 pgsql-16beta2
+ # where the line marked with a * is the currently active version
+ #
+ # we are only interested in the first word of each line, which is the version number
+ # thus we strip the whitespace and the * from the line and split it into words
+ # and take the first word
+ versions = [line.strip('* ').split()[0] for line in output.decode('utf-8').splitlines()]
+
+ # create the list of configurations per version
+ configurations = []
+ for version in versions:
+ configurations.append(generate_configuration(version))
+
+ # create the json file
+ c_cpp_properties = {
+ "configurations": configurations,
+ "version": 4
+ }
+
+ # write the c_cpp_properties.json file
+ with open(target_path, 'w') as f:
+ json.dump(c_cpp_properties, f, indent=4)
+
+
+def generate_configuration(version):
+ """Returns a configuration for the given postgres version.
+
+ >>> generate_configuration('14.8')
+ {
+ "name": "Citus Development Configuration - Postgres 14.8",
+ "includePath": [
+ "/usr/local/include",
+ "/home/citus/.pgenv/src/postgresql-14.8/src/**",
+ "${workspaceFolder}/**",
+ "${workspaceFolder}/src/include/",
+ ],
+ "configurationProvider": "ms-vscode.makefile-tools"
+ }
+ """
+ return {
+ "name": f"Citus Development Configuration - Postgres {version}",
+ "includePath": [
+ "/usr/local/include",
+ f"/home/citus/.pgenv/src/postgresql-{version}/src/**",
+ "${workspaceFolder}/**",
+ "${workspaceFolder}/src/include/",
+ ],
+ "configurationProvider": "ms-vscode.makefile-tools"
+ }
+
+
+if __name__ == '__main__':
+ arguments = docopt(__doc__, version='0.1.0')
+ main(arguments)

.devcontainer/.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Attach Citus (devcontainer)",
+ "type": "cppdbg",
+ "request": "attach",
+ "processId": "${command:pickProcess}",
+ "program": "/home/citus/.pgenv/pgsql/bin/postgres",
+ "additionalSOLibSearchPath": "/home/citus/.pgenv/pgsql/lib",
+ "setupCommands": [
+ {
+ "text": "handle SIGUSR1 noprint nostop pass",
+ "description": "let gdb not stop when SIGUSR1 is sent to process",
+ "ignoreFailures": true
+ }
+ ],
+ },
+ ]
+}

.devcontainer/Dockerfile

@@ -0,0 +1,212 @@
+FROM ubuntu:22.04 AS base
+
+# environment is to make python pass an interactive shell, probably not the best timezone given a wide variety of colleagues
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# install build tools
+RUN apt update && apt install -y \
+ bzip2 \
+ cpanminus \
+ curl \
+ flex \
+ gcc \
+ git \
+ libcurl4-gnutls-dev \
+ libicu-dev \
+ libkrb5-dev \
+ liblz4-dev \
+ libpam0g-dev \
+ libreadline-dev \
+ libselinux1-dev \
+ libssl-dev \
+ libxslt-dev \
+ libzstd-dev \
+ locales \
+ make \
+ perl \
+ pkg-config \
+ python3 \
+ python3-pip \
+ software-properties-common \
+ sudo \
+ uuid-dev \
+ valgrind \
+ zlib1g-dev \
+ && add-apt-repository ppa:deadsnakes/ppa -y \
+ && apt install -y \
+ python3.9-full \
+ && apt clean
+
+RUN sudo pip3 install pipenv pipenv-shebang
+
+RUN cpanm install IPC::Run
+
+RUN locale-gen en_US.UTF-8
+
+# add the citus user to sudoers and allow all sudoers to login without a password prompt
+RUN useradd -ms /bin/bash citus \
+ && usermod -aG sudo citus \
+ && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+WORKDIR /home/citus
+USER citus
+
+# run all make commands with the number of cores available
+RUN echo "export MAKEFLAGS=\"-j \$(nproc)\"" >> "/home/citus/.bashrc"
+
+RUN git clone --branch v1.3.2 --depth 1 https://github.com/theory/pgenv.git .pgenv
+COPY --chown=citus:citus pgenv/config/ .pgenv/config/
+ENV PATH="/home/citus/.pgenv/bin:${PATH}"
+ENV PATH="/home/citus/.pgenv/pgsql/bin:${PATH}"
+
+USER citus
+
+# build postgres versions separately for effective parrallelism and caching of already built versions when changing only certain versions
+FROM base AS pg14
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.9
+RUN rm .pgenv/src/*.tar*
+RUN make -C .pgenv/src/postgresql-*/ clean
+RUN make -C .pgenv/src/postgresql-*/src/include install
+
+# create a staging directory with all files we want to copy from our pgenv build
+# we will copy the contents of the staged folder into the final image at once
+RUN mkdir .pgenv-staging/
+RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
+RUN rm .pgenv-staging/config/default.conf
+
+FROM base AS pg15
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.4
+RUN rm .pgenv/src/*.tar*
+RUN make -C .pgenv/src/postgresql-*/ clean
+RUN make -C .pgenv/src/postgresql-*/src/include install
+
+# create a staging directory with all files we want to copy from our pgenv build
+# we will copy the contents of the staged folder into the final image at once
+RUN mkdir .pgenv-staging/
+RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
+RUN rm .pgenv-staging/config/default.conf
+
+FROM base AS pg16
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.0
+RUN rm .pgenv/src/*.tar*
+RUN make -C .pgenv/src/postgresql-*/ clean
+RUN make -C .pgenv/src/postgresql-*/src/include install
+
+# create a staging directory with all files we want to copy from our pgenv build
+# we will copy the contents of the staged folder into the final image at once
+RUN mkdir .pgenv-staging/
+RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
+RUN rm .pgenv-staging/config/default.conf
+
+FROM base AS uncrustify-builder
+
+RUN sudo apt update && sudo apt install -y cmake tree
+
+WORKDIR /uncrustify
+RUN curl -L https://github.com/uncrustify/uncrustify/archive/uncrustify-0.68.1.tar.gz | tar xz
+WORKDIR /uncrustify/uncrustify-uncrustify-0.68.1/
+RUN mkdir build
+WORKDIR /uncrustify/uncrustify-uncrustify-0.68.1/build/
+RUN cmake ..
+RUN make -sj8
+
+RUN make install DESTDIR=/uncrustify
+
+# builder for all pipenv's to get them contained in a single layer
+FROM base AS pipenv
+
+WORKDIR /workspaces/citus/
+
+# tools to sync pgenv with vscode
+COPY --chown=citus:citus .vscode/Pipfile .vscode/Pipfile.lock .devcontainer/.vscode/
+RUN ( cd .devcontainer/.vscode && pipenv install )
+
+# environment to run our failure tests
+COPY --chown=citus:citus src/ src/
+RUN ( cd src/test/regress && pipenv install )
+
+# assemble the final container by copying over the artifacts from separately build containers
+FROM base AS devcontainer
+
+LABEL org.opencontainers.image.source=https://github.com/citusdata/citus
+LABEL org.opencontainers.image.description="Development container for the Citus project"
+LABEL org.opencontainers.image.licenses=AGPL-3.0-only
+
+RUN yes | sudo unminimize
+
+# install developer productivity tools
+RUN sudo apt update \
+ && sudo apt install -y \
+ autoconf2.69 \
+ bash-completion \
+ fswatch \
+ gdb \
+ htop \
+ libdbd-pg-perl \
+ libdbi-perl \
+ lsof \
+ man \
+ net-tools \
+ pspg \
+ tree \
+ vim \
+ && sudo apt clean
+
+# Since gdb will run in the context of the root user when debugging citus we will need to both
+# download the gdbpg.py script as the root user, into their home directory, as well as add .gdbinit
+# as a file owned by root
+# This will make that as soon as the debugger attaches to a postgres backend (or frankly any other process)
+# the gdbpg.py script will be sourced and the developer can direcly use it.
+RUN sudo curl -o /root/gdbpg.py https://raw.githubusercontent.com/tvesely/gdbpg/6065eee7872457785f830925eac665aa535caf62/gdbpg.py
+COPY --chown=root:root .gdbinit /root/
+
+# add some common tools to the final container
+# bin directory for user tools
+RUN mkdir .bin
+ENV PATH="/home/citus/.bin:${PATH}"
+
+# for persistent bash history across devcontainers we need to have
+# a) a directory to store the history in
+# b) a prompt command to append the history to the file
+# c) specify the history file to store the history in
+# b and c are done in the .bashrc to make it persistent across shells only
+RUN sudo install -d -o citus -g citus /commandhistory \
+ && echo "export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" >> "/home/citus/.bashrc"
+
+# install citus-dev
+RUN git clone --branch develop https://github.com/citusdata/tools.git citus-tools \
+ && ( cd citus-tools/citus_dev && pipenv install ) \
+ && ln -s /home/citus/citus-tools/citus_dev/citus_dev-pipenv .bin/citus_dev \
+ && sudo make -C citus-tools/uncrustify install bindir=/usr/local/bin pkgsysconfdir=/usr/local/etc/ \
+ && mkdir -p ~/.local/share/bash-completion/completions/ \
+ && ln -s ~/citus-tools/citus_dev/bash_completion ~/.local/share/bash-completion/completions/citus_dev
+
+# TODO some LC_ALL errors, possibly solved by locale-gen
+RUN git clone https://github.com/so-fancy/diff-so-fancy.git \
+ && ln -s /home/citus/diff-so-fancy/diff-so-fancy .bin/
+
+COPY --link --from=uncrustify-builder /uncrustify/usr/ /usr/
+
+COPY --link --from=pg14 /home/citus/.pgenv-staging/ /home/citus/.pgenv/
+COPY --link --from=pg15 /home/citus/.pgenv-staging/ /home/citus/.pgenv/
+COPY --link --from=pg16 /home/citus/.pgenv-staging/ /home/citus/.pgenv/
+
+COPY --link --from=pipenv /home/citus/.local/share/virtualenvs/ /home/citus/.local/share/virtualenvs/
+
+# place to run your cluster with citus_dev
+VOLUME /data
+RUN sudo mkdir /data \
+ && sudo chown citus:citus /data
+
+COPY --chown=citus:citus .psqlrc .
+
+# with the copy linking of layers github actions seem to misbehave with the ownership of the
+# directories leading upto the link, hence a small patch layer to have to right ownerships set
+RUN sudo chown --from=root:root citus:citus -R ~
+
+# sets default pg version
+RUN pgenv switch 16.0
+
+# make connecting to the coordinator easy
+ENV PGPORT=9700

.devcontainer/Makefile

@@ -0,0 +1,11 @@
+
+init: ../.vscode/c_cpp_properties.json ../.vscode/launch.json
+
+../.vscode:
+ mkdir -p ../.vscode
+
+../.vscode/launch.json: ../.vscode .vscode/launch.json
+ cp .vscode/launch.json ../.vscode/launch.json
+
+../.vscode/c_cpp_properties.json: ../.vscode
+ ./.vscode/generate_c_cpp_properties-json.py ../.vscode/c_cpp_properties.json