Gh-334: Smoother gaffer gremlin deployment (#337)

* add basic proof of concept using proxy store

* tidy and hook into existing CI

* readme updates

* add to publish ci

* merge existing demo image with new image

* move build to docker compose and tidy

---------

Co-authored-by: GCHQDeveloper314 <94527357+GCHQDeveloper314@users.noreply.github.com>

.github/workflows/continuous-integration.yaml

@@ -58,8 +58,8 @@ jobs:
       run: ./cd/lint_charts.sh
 
     - name: Deploy to Kubernetes
-      run: ./cd/deploy_to_kind.sh
-  
+      run: ./cd/deploy_to_kind.sh ./docker/accumulo2.env
+
     - name: Run gaffer-road-traffic Tests
       run: helm test gaffer || (kubectl get po && kubectl describe po && kubectl logs -l app.kubernetes.io/component=test --tail=-1 && df -h && false)
 

.gitignore

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,3 +21,4 @@
 **/.DS_Store
 kubernetes/*/Chart.lock
 kubernetes/*/charts/
+target/

cd/build_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,31 +18,36 @@
 # Required for variables from sourced env file to automatically be visible to docker compose.
 set -e -a
 
-root_directory="$( cd $(dirname $(dirname $0)) > /dev/null 2>&1 && pwd )"
-cd $root_directory
+ROOT_DIR="$(readlink -f "$(dirname "$(dirname "${0}")")")"
 
-if [ ! -z "$1" ]; then
-	ENV_FILE=$1
-else
-	echo "Error - Environment file not set"; exit 1;
-fi
+pushd "${ROOT_DIR}" || exit 1
 
-# The following command sets:
+# The following env file will be sourced to set:
 # HADOOP_VERSION
 # GAFFER_VERSION
 # GAFFERPY_VERSION
 # ACCUMULO_VERSION
 # SPARK_VERSION
-source "${ENV_FILE}"
+# TINKERPOP_VERSION
+if [[ -f "${1}" ]]; then
+    source "${1}"
+else
+    echo "Error - Environment file not set"
+    exit 1
+fi
 
 # Builds all of the Gaffer and Accumulo related images:
 docker compose --project-directory ./docker/accumulo/ -f ./docker/accumulo/docker-compose.yaml build
 docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./docker/gaffer-road-traffic-loader/docker-compose.yaml build
 # Builds all of the notebook related images:
 docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook
 docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build
+# Builds the Gaffer Gremlin server
+./docker/gaffer-gremlin/build.sh
 
 # Set $JHUB_OPTIONS_SERVER_VERSION
 source ./docker/gaffer-jhub-options-server/get-version.sh
 # Builds the jhub options server:
 docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build
+
+popd || exit 1

cd/deploy_to_kind.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,7 +22,13 @@ kind create cluster --quiet --config ./cd/kind.yaml --image kindest/node:v1.24.4
 # HADOOP_VERSION
 # GAFFER_VERSION
 # SPARK_VERSION
-source ./docker/gaffer-pyspark-notebook/.env
+# KUBECTL_VERSION
+if [[ -f "${1}" ]]; then
+    source "${1}"
+else
+    echo "Error - Environment file not set"
+    exit 1
+fi
 # JHUB_OPTIONS_SERVER_VERSION
 source ./docker/gaffer-jhub-options-server/get-version.sh
 

cd/publish_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -101,6 +101,7 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
 pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}"
+pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}"
 pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}"
 pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}"
 pushContainer gchq/spark-py "${SPARK_VERSION}"

cd/update_versions.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,9 +23,9 @@ fi
 # GAFFER_VERSION
 # GAFFERPY_VERSION
 # SPARK_VERSION
-source ./docker/gaffer-pyspark-notebook/.env
+source ./docker/accumulo2.env
 # JHUB_OPTIONS_SERVER_VERSION
-source docker/gaffer-jhub-options-server/get-version.sh
+source ./docker/gaffer-jhub-options-server/get-version.sh
 
 # hdfs
 [ ! -z "${APP_VERSION}" ] && yq eval ".version = \"${APP_VERSION}\"" -i ./kubernetes/hdfs/Chart.yaml

docker/accumulo2.env

@@ -7,3 +7,5 @@ GAFFER_VERSION=2.1.0
 GAFFER_TESTER_VERSION=2.1.0
 GAFFERPY_VERSION=2.1.0
 SPARK_VERSION=3.1.2
+TINKERPOP_VERSION=3.7.1
+KUBECTL_VERSION=1.23.0

docker/gaffer-gremlin/Dockerfile

@@ -0,0 +1,28 @@
+# Copyright 2023-2024 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
+ARG BASE_IMAGE_TAG=3.7.1
+
+# Base image provides WORKDIR, USER and ENTRYPOINT
+FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
+
+# Copy JARS
+COPY ./target/dependency/*.jar ext/gafferpop/plugin/
+
+# Copy configs
+COPY ./conf/ conf/
+
+# Set server to run
+CMD ["conf/gaffer-gremlin-server.yaml"]

docker/gaffer-gremlin/README.md

@@ -0,0 +1,56 @@
+# gaffer-gremlin
+
+## Build
+
+Run the supplied `build.sh` script with the required environment vars (can
+source `accumolo2.env` file) to pull all dependencies and build the container.
+
+_Note requires Maven and Docker_
+
+Optionally run each command separately to configure the containers tags etc.
+
+## Configure
+
+The container will use the Gaffer Proxy store by default to connect to an
+existing graph and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)).
+
+The Gaffer graph the container will connect to can be configured as usual by
+editing the `store.properties` file, which you can also bind mount over on an
+existing image. The config file locations are under the predefined workdir
+set by the parent gremlin server image the key locations in the image are:
+
+- `/opt/gremlin-server/conf/gaffer/store.properties` - Override for custom store properties.
+- `/opt/gremlin-server/conf/gafferpop/gafferpop.properties` - Override to configure the graph.
+
+The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml`
+this again can be modified as needed or bind mounted over. Please see the
+[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/)
+for more information on configuring this image.
+
+## Run
+
+Simply run the container to publish the Gremlin server making sure to make the
+configured port available (port 8182 by default) this can then be connected to
+via the address specified in the server's yaml config to use Gremlin traversal.
+The server can then be connected via the Gremlin console e.g. if using
+`gremlinpython`:
+
+```python
+from gremlin_python.process.anonymous_traversal_source import traversal
+
+g = traversal().withRemote(
+    DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))
+```
+
+### Demo Deployment
+
+A demo/example using the tinkerpop 'modern' dataset and accumulo backed Gaffer
+is available under the `example` directory. This can be ran using docker compose
+to deploy the containers then, the provided jupiter notebook demonstrates how to
+connect and some basic queries on the data using `gremlinpython`.
+
+Run the example with:
+
+```bash
+docker compose up
+```

docker/gaffer-gremlin/build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright 2023-2024 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")"
+
+# Check if already have env var for version
+[[ -z "${GAFFER_VERSION}" ]] && \
+    echo "Missing GAFFER_VERSION env var" && \
+    exit 1
+[[ -z "${TINKERPOP_VERSION}" ]] && \
+    echo "Missing TINKERPOP_VERSION env var" && \
+    exit 1
+
+# Build from relevant directory
+pushd "${SCRIPT_DIR}" || exit 1
+    # Download JARs
+    mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}"
+
+    # Build container
+    docker compose build
+
+    # Clean
+    mvn clean
+popd || exit 1
+
+echo "Build Successful"

docker/gaffer-gremlin/compose.yaml

@@ -0,0 +1,25 @@
+# Copyright 2024 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.7"
+
+services:
+  gaffer-gremlin:
+    image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}
+    build:
+      context: .
+      args:
+        BASE_IMAGE_TAG: ${TINKERPOP_VERSION}
+    ports:
+    - 8182:8182

docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml → docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,21 +16,27 @@ host: localhost
 port: 8182
 evaluationTimeout: 30000
 graphs: {
-  graph: conf/gafferpop/gafferpop-tinkerpop-modern.properties}
+  graph: conf/gafferpop/gafferpop.properties
+}
 scriptEngines: {
   gremlin-groovy: {
     plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
                uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {},
                org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true},
                org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
-               org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}}
+               org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}
+    }
   }
 }
 serializers:
-  - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}                                                                 # application/vnd.graphbinary-v1.0-stringd
+  # application/vnd.graphbinary-v1.0
+  - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 }
+  # application/vnd.graphbinary-v1.0-stringd
+  - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
 
 metrics: {
-  slf4jReporter: {enabled: true, interval: 180000}}
+  slf4jReporter: {enabled: true, interval: 180000}
+}
 strictTransactionManagement: false
 idleConnectionTimeout: 0
 keepAliveInterval: 0

docker/gaffer-gremlin/conf/gaffer/store.properties

@@ -0,0 +1,20 @@
+#
+# Copyright 2023-2024 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore
+# These can be configured to an existing graph deployment
+gaffer.host=localhost
+gaffer.port=8080
+gaffer.context-root=/rest/latest

docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties → docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties

@@ -1,5 +1,5 @@
 #
-# Copyright 2016-2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,6 @@
 #
 gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
 gaffer.graphId=graph1
-gaffer.storeproperties=conf/gafferpop/gaffer/store.properties
-gaffer.schemas=conf/gafferpop/gaffer/schema/
+gaffer.storeproperties=conf/gaffer/store.properties
+gaffer.schemas=conf/gaffer/schema/
 gaffer.userId=user01