Skip to content

Commit

Permalink
Merge commit '59e29d03f89d3966d6210154c173d48eec22799d' into dev
Browse files Browse the repository at this point in the history
  • Loading branch information
crazyzlj committed Jun 11, 2024
2 parents 568edeb + 59e29d0 commit 7e3af8f
Show file tree
Hide file tree
Showing 12 changed files with 245 additions and 303 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,9 @@ jobs:
- name: Setup xcode
uses: maxim-lobanov/setup-xcode@v1

# Default MongoBD was preinstalled on macos-12 and macos-11
# see https://github.com/actions/runner-images/tree/main/images/macos
#
# default mongod install path: /usr/local/bin/mongod
# default mongod.conf: /usr/local/etc/mongod.conf
# systemLog:
Expand All @@ -173,12 +176,22 @@ jobs:
# dbPath: /usr/local/var/mongodb
# net:
# bindIp: 127.0.0.1
- name: Check mongod
run: |
which mongod
cat /usr/local/etc/mongod.conf
- name: Start mongod manually
run: mongod --config /usr/local/etc/mongod.conf --fork
#
# - name: Check mongod
# run: |
# which mongod
# cat /usr/local/etc/mongod.conf
# - name: Start mongod manually
# run: mongod --config /usr/local/etc/mongod.conf --fork

# https://github.com/ankane/setup-mongodb
- name: Setup MongoDB
uses: ankane/setup-mongodb@v1
with:
mongodb-version: 7.0

- name: Run MongoDB
run: mongosh --eval "db.version()"

- name: Install mongo-c-driver
run: brew install mongo-c-driver
Expand Down
25 changes: 19 additions & 6 deletions seims/src/ccgl/.github/workflows/cmake_builds_with-mongodb.yml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,9 @@ jobs:
- name: Setup xcode
uses: maxim-lobanov/setup-xcode@v1

# Default MongoBD was preinstalled on macos-12 and macos-11
# see https://github.com/actions/runner-images/tree/main/images/macos
#
# default mongod install path: /usr/local/bin/mongod
# default mongod.conf: /usr/local/etc/mongod.conf
# systemLog:
Expand All @@ -129,12 +132,22 @@ jobs:
# dbPath: /usr/local/var/mongodb
# net:
# bindIp: 127.0.0.1
- name: Check mongod
run: |
which mongod
cat /usr/local/etc/mongod.conf
- name: Start mongod manually
run: mongod --config /usr/local/etc/mongod.conf --fork
#
# - name: Check mongod
# run: |
# which mongod
# cat /usr/local/etc/mongod.conf
# - name: Start mongod manually
# run: mongod --config /usr/local/etc/mongod.conf --fork

# https://github.com/ankane/setup-mongodb
- name: Setup MongoDB
uses: ankane/setup-mongodb@v1
with:
mongodb-version: 7.0

- name: Run MongoDB
run: mongosh --eval "db.version()"

- name: Install mongo-c-driver
run: brew install mongo-c-driver
Expand Down
124 changes: 124 additions & 0 deletions seims/src/ccgl/.github/workflows/deploy_images.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
name: Create and publish CCGL Docker images

# Configures this workflow to run every time a change is pushed to the branches.
on:
push:
branches:
- 'master'
- 'dev'

# Defines custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
# IMAGE_NAME: ${{ github.repository }} ==> crazyzlj/ccgl
IMAGE_NAME: ccgl
# github container registry
REGISTRY: ghcr.io
NAMESPACE: ${{ github.repository_owner }}
# aliyun ACR
REGION_ID_ACR: cn-hangzhou
REGISTRY_ACR: registry.cn-hangzhou.aliyuncs.com
NAMESPACE_ACR: ljzhu-geomodels
# IMAGE_TAG: ${{ github.sha }}

# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
# permissions: write-all
permissions:
contents: read
packages: write
attestations: write
id-token: write
#
steps:
- name: Checkout repository
uses: actions/checkout@v4

# login alibaba Container Registry
- name: Login to ACR
uses: aliyun/acr-login@v1
with:
login-server: https://registry.cn-hangzhou.aliyuncs.com
region-id: cn-hangzhou
username: "${{ secrets.ACR_USERNAME }}"
password: "${{ secrets.ACR_PASSWORD }}"

# login ghcr.io
- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY_ACR }}/${{ env.NAMESPACE_ACR }}/${{ env.IMAGE_NAME }}
${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.IMAGE_NAME }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build basic CCGL
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile-alpine
push: true
platforms: linux/amd64,linux/arm64
tags: |
${{ env.REGISTRY_ACR }}/${{ env.NAMESPACE_ACR }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine
${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Build CCGL with gdal
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile-alpine-with-gdal
push: true
platforms: linux/amd64,linux/arm64
tags: |
${{ env.REGISTRY_ACR }}/${{ env.NAMESPACE_ACR }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine-with-gdal
${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine-with-gdal
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Build CCGL with mongodb
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile-alpine-with-mongodb
push: true
platforms: linux/amd64,linux/arm64
tags: |
${{ env.REGISTRY_ACR }}/${{ env.NAMESPACE_ACR }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine-with-mongodb
${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine-with-mongodb
cache-from: type=gha
cache-to: type=gha,mode=max

- name: Build CCGL with gdal and mongodb
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile-alpine-with-gdal-mongodb
push: true
platforms: linux/amd64,linux/arm64
tags: |
${{ env.REGISTRY_ACR }}/${{ env.NAMESPACE_ACR }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine-with-gdal-mongodb
${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-alpine-with-gdal-mongodb
cache-from: type=gha
cache-to: type=gha,mode=max
45 changes: 14 additions & 31 deletions seims/src/ccgl/docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,38 +15,21 @@ docker build --progress=plain -t crazyzlj/ccgl:<tag>-test -f docker/test/<tag>/D

### CCGL with the support of MongoDB

See the [official tutorial](https://www.mongodb.com/compatibility/docker).

+ Select a proper tag of the official mongo image in [supported-tags-and-respective-dockerfile-links](https://github.com/docker-library/docs/blob/master/mongo/README.md#supported-tags-and-respective-dockerfile-links), e.g., `4.4.14-focal`.
+ Start a mongo server instance: `docker run --name some-mongo -d -p <client_port>:27017 mongo:tag`,
where `some-mongo` is the name you want to assign to your container and
`tag` is the tag specifying the MongoDB version you want,
`client_port` is the port you want to exposed to your localhost, e.g., `27017` or any other legal number.
For example,
+ `docker network create docker-mongo-network`
+ `docker run -d -p 27018:27017 --name mongodb-docker mongo:4.4.14-focal`
+ `docker ps` will show something like:
```
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
6cef37edee6a mongo:4.4.14-focal "docker-entrypoint.s…" 5 seconds ago Up 4 seconds 0.0.0.0:27018->27017/tcp mongodb-docker
```
See the [official tutorial: Docker and MongoDB](https://www.mongodb.com/resources/products/compatibilities/docker).

+ Select a proper tag of the official [mongodb-community-server image](https://hub.docker.com/r/mongodb/mongodb-community-server), e.g., `mongodb/mongodb-community-server:4.4.28-ubuntu2004`.
+ Create a docker network for the MongoDB server and other docker applications who want use this MongoDB: `docker network create docker-mongo-network`
+ Start a mongodb server under the above docker network: `docker run -d --rm --name mongodb-docker --network docker-mongo-network -p <client_port>:27017 -v </path/to/store/mongodb/data>:/data/db mongodb/mongodb-community-server:4.4.28-ubuntu2004 mongod --bind_ip localhost,mongodb-docker`
+ `</path/to/store/mongodb/data>` means the local directory (path in English and without spaces) to store MongoDB data
+ `client_port` is the port you want to exposed to your localhost, e.g., `27017` or any other legal number.
+ Run `docker logs mongodb-docker` to check the logs which provide a wealth of useful information.
+ Now, you can connect to the MongoDB running in the container from *the host* host via `localhost:27018`.
+ Mount data directory of host using `-v </path/to/any>:/data/db`, e.g.,
`docker run -d -v /Users/ljzhu/Documents/data/docker_mongodata:/data/db -p 27017:27017 --name mongodb-docker mongo:4.4.14-focal`
+ There are couple of ways to do [Connect to MongoDB from another Docker container](https://github.com/docker-library/docs/blob/master/mongo/README.md#connect-to-mongodb-from-another-docker-container).
Also refers to one userful answer in [stackflow](https://stackoverflow.com/a/43962099).
+ (Currently tested) Use mongodb container ip address: `docker inspect -f '{{.NetworkSettings.IPAddress}}' mongo-docker`.
The command will output the IP of the mongodb container which can be used in another application's container.
For example, `./unittestd -host 172.17.0.3 -port 27017`
The IP of the mongodb container can be passed as an argument when build another docker image,
also see [here](https://stackoverflow.com/a/34254700):
`docker build --progress=plain -t crazyzlj/ccgl:alpine-with-mongodb-test --build-arg host=172.17.0.2 --build-arg port=27017 -f docker/test/alpine-with-mongodb/Dockerfile .`
+ ...
+ Now, you can connect to the MongoDB running in the container from *the host* host via `localhost:<client_port>` (e.g., using Robo 3T, VSCode with MongoDB plugin).
+ Check the IP address of the MongoDB container within the above defined docker network: `docker inspect network docker-mongo-network`, e.g., `172.18.0.2`
+ Run the `ccgl` container within the same docker network and pass the IP address as arguments: `docker run --rm -v $PWD/data:/data --net docker-mongo-network registry.cn-hangzhou.aliyuncs.com/ljzhu-geomodels/ccgl:dev-alpine-with-gdal-mongodb mask_rasterio -in /data/raster/dem_1.tif -out gfs dem_1 -mongo 172.18.0.2 27017 test spatial`.

> References:
> + [Connect to MongoDB from another Docker container](https://github.com/docker-library/docs/blob/master/mongo/README.md#connect-to-mongodb-from-another-docker-container).
> + An useful answer in [stackflow](https://stackoverflow.com/a/43962099).
## Release docker images

Expand Down
41 changes: 19 additions & 22 deletions seims/src/ccgl/docker/alpine-with-gdal-mongodb/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,28 +1,25 @@
##
# crazyzlj/ccgl:alpine-with-gdal-mongodb
#
# This script is written based on PROJ and GDAL.
# This script is written based on PROJ and GDAL.
# https://github.com/OSGeo/PROJ/blob/master/Dockerfile
# https://github.com/OSGeo/gdal/tree/master/docker
#
# Usage:
# Usage:
# > cd CCGL
# > docker build -t <tag> -f docker/apline-with-gdal-mongodb/Dockerfile .
#
# Copyright 2022 Liang-Jun Zhu <zlj@lreis.ac.cn>
# Copyright 2022-2024 Liang-Jun Zhu <zlj@lreis.ac.cn>

# Use GDAL image with the tag of alpine-small-latest as the build and test container
ARG GDAL_VERSION=latest
FROM osgeo/gdal:alpine-small-${GDAL_VERSION} as builder
# Use GDAL image tagged alpine-small-latest as the build and test container
FROM ghcr.io/osgeo/gdal:alpine-small-latest as builder

LABEL maintainer="Liang-Jun Zhu <zlj@lreis.ac.cn>"

# Replace alpine repository source cdn
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && apk update && apk upgrade

# Setup build env
RUN apk add --no-cache cmake make g++ mongo-c-driver-dev
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
RUN apk update && apk upgrade && \
apk add --no-cache cmake make g++ mongo-c-driver-dev

# Copy source directory
WORKDIR /CCGL
Expand All @@ -44,18 +41,18 @@ RUN cd /CCGL \
&& cd ..

# # Build final image
FROM osgeo/gdal:alpine-small-${GDAL_VERSION} as runner
FROM ghcr.io/osgeo/gdal:alpine-small-latest as runner

# Replace alpine repository source cdn
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && apk update && apk upgrade

# Add OpenMP library for running apps based on CCGL
RUN apk add --no-cache libgomp mongo-c-driver
# Add gnu standard C++ library and OpenMP library for running apps based on CCGL
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
RUN apk update && apk upgrade && \
apk add --no-cache libstdc++ libgomp mongo-c-driver

# Order layers starting with less frequently varying ones
ARG INSTALL_DIR=/CCGL/dist
COPY --from=builder ${INSTALL_DIR}/bin/ /usr/bin/
COPY --from=builder ${INSTALL_DIR}/include/ /usr/include/
COPY --from=builder ${INSTALL_DIR}/lib/ /usr/lib/
COPY --from=builder ${INSTALL_DIR}/share/ccgl/ /usr/share/ccgl/
COPY --from=builder ${INSTALL_DIR}/bin/ /usr/local/bin/
COPY --from=builder ${INSTALL_DIR}/include/ /usr/local/include/
COPY --from=builder ${INSTALL_DIR}/lib/ /usr/local/lib/
COPY --from=builder ${INSTALL_DIR}/share/ccgl/ /usr/local/share/ccgl/
41 changes: 19 additions & 22 deletions seims/src/ccgl/docker/alpine-with-gdal/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,28 +1,25 @@
##
# crazyzlj/ccgl:alpine-with-gdal
#
# This script is written based on PROJ and GDAL.
# This script is written based on PROJ and GDAL.
# https://github.com/OSGeo/PROJ/blob/master/Dockerfile
# https://github.com/OSGeo/gdal/tree/master/docker
#
# Usage:
# Usage:
# > cd CCGL
# > docker build -t <tag> -f docker/apline-with-gdal/Dockerfile .
#
# Copyright 2022 Liang-Jun Zhu <zlj@lreis.ac.cn>
# Copyright 2022-2024 Liang-Jun Zhu <zlj@lreis.ac.cn>

# Use GDAL image with the tag of alpine-small-latest as the build and test container
ARG GDAL_VERSION=latest
FROM osgeo/gdal:alpine-small-${GDAL_VERSION} as builder
# Use GDAL image tagged alpine-small-latest as the build and test container
FROM ghcr.io/osgeo/gdal:alpine-small-latest as builder

LABEL maintainer="Liang-Jun Zhu <zlj@lreis.ac.cn>"

# Replace alpine repository source cdn
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && apk update && apk upgrade

# Setup build env
RUN apk add --no-cache cmake make g++
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
RUN apk update && apk upgrade && \
apk add --no-cache cmake make g++

# Copy source directory
WORKDIR /CCGL
Expand All @@ -44,18 +41,18 @@ RUN cd /CCGL \
&& cd ..

# # Build final image
FROM osgeo/gdal:alpine-small-${GDAL_VERSION} as runner
FROM ghcr.io/osgeo/gdal:alpine-small-latest as runner

# Replace alpine repository source cdn
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && apk update && apk upgrade

# Add OpenMP library for running apps based on CCGL
RUN apk add --no-cache libgomp
# Add gnu standard C++ library and OpenMP library for running apps based on CCGL
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
RUN apk update && apk upgrade && \
apk add --no-cache libstdc++ libgomp

# Order layers starting with less frequently varying ones
ARG INSTALL_DIR=/CCGL/dist
COPY --from=builder ${INSTALL_DIR}/bin/ /usr/bin/
COPY --from=builder ${INSTALL_DIR}/include/ /usr/include/
COPY --from=builder ${INSTALL_DIR}/lib/ /usr/lib/
COPY --from=builder ${INSTALL_DIR}/share/ccgl/ /usr/share/ccgl/
COPY --from=builder ${INSTALL_DIR}/bin/ /usr/local/bin/
COPY --from=builder ${INSTALL_DIR}/include/ /usr/local/include/
COPY --from=builder ${INSTALL_DIR}/lib/ /usr/local/lib/
COPY --from=builder ${INSTALL_DIR}/share/ccgl/ /usr/local/share/ccgl/
Loading

0 comments on commit 7e3af8f

Please sign in to comment.