From 626f4237a1652b2ebe7dbd180a88bd63d3ea1a37 Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Fri, 8 Dec 2023 16:01:31 +0000
Subject: [PATCH 01/12] add basic proof of concept using proxy store
---
docker/gaffer-gremlin/Dockerfile | 27 ++++++++++
docker/gaffer-gremlin/README..md | 28 ++++++++++
docker/gaffer-gremlin/build.sh | 30 +++++++++++
.../conf/gaffer-gremlin-server.yaml | 45 ++++++++++++++++
.../conf/gaffer/store.properties | 20 +++++++
.../conf/gafferpop/gafferpop.properties | 19 +++++++
docker/gaffer-gremlin/pom.xml | 52 +++++++++++++++++++
7 files changed, 221 insertions(+)
create mode 100644 docker/gaffer-gremlin/Dockerfile
create mode 100644 docker/gaffer-gremlin/README..md
create mode 100755 docker/gaffer-gremlin/build.sh
create mode 100644 docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
create mode 100644 docker/gaffer-gremlin/conf/gaffer/store.properties
create mode 100644 docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
create mode 100644 docker/gaffer-gremlin/pom.xml
diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile
new file mode 100644
index 00000000..30917f97
--- /dev/null
+++ b/docker/gaffer-gremlin/Dockerfile
@@ -0,0 +1,27 @@
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
+ARG BASE_IMAGE_TAG=3.7.1
+
+FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
+
+# Copy JARS
+COPY ./target/dependency/*.jar ext/gafferpop/plugin/
+
+# Copy configs
+COPY ./conf/ conf/
+
+# Set server to run
+CMD ["conf/gaffer-gremlin-server.yaml"]
diff --git a/docker/gaffer-gremlin/README..md b/docker/gaffer-gremlin/README..md
new file mode 100644
index 00000000..7a1862bc
--- /dev/null
+++ b/docker/gaffer-gremlin/README..md
@@ -0,0 +1,28 @@
+# gaffer-gremlin
+
+## Build
+
+Run the supplied `build.sh` script to pull all dependencies and build the
+container.
+
+_Note requires Maven and Docker_
+
+Optionally run each command separately to configure the containers tags etc.
+
+## Configure
+
+The container will use the Gaffer Proxy store to connect to an existing graph
+and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/3.7.1/reference/#connecting-gremlin-server)).
+
+The Gaffer graph the container will connect to can be configured as usual by
+editing the `store.properties` file, which you can also bind mount over on an
+existing image.
+
+The configuration for the Gremlin server is provided by the `[gaffer-gremlin-server.yaml](./conf/gaffer-gremlin-server.yaml)`
+this again can be modified as needed or bind mounted over.
+
+## Run
+
+Simply run the container to publish the Gremlin server this can then be
+connected to via the address specified in the server's yaml config to use
+Gremlin traversal.
diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh
new file mode 100755
index 00000000..866b924f
--- /dev/null
+++ b/docker/gaffer-gremlin/build.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare CONTAINER_VER="2.1.0"
+declare CURRENT_DIR="$(dirname "${0}")"
+
+pushd "${CURRENT_DIR}"
+ # Download JARs
+ mvn clean dependency:copy-dependencies
+
+ # Build container
+ docker build -t gaffer-gremlin:"${CONTAINER_VER}" .
+
+ # Clean
+ mvn clean
+popd
+
+echo "Build Successful"
diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
new file mode 100644
index 00000000..6dd17d7a
--- /dev/null
+++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
@@ -0,0 +1,45 @@
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+host: localhost
+port: 8182
+evaluationTimeout: 30000
+graphs: {
+ graph: conf/gafferpop/gafferpop.properties
+}
+scriptEngines: {
+ gremlin-groovy: {
+ plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
+ uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {},
+ org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true},
+ org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
+ org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}
+ }
+ }
+}
+serializers:
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
+
+metrics: {
+ slf4jReporter: {enabled: true, interval: 180000}}
+strictTransactionManagement: false
+idleConnectionTimeout: 0
+keepAliveInterval: 0
+maxInitialLineLength: 4096
+maxHeaderSize: 8192
+maxChunkSize: 8192
+maxContentLength: 10485760
+maxAccumulationBufferComponents: 1024
+resultIterationBatchSize: 64
+enableAuditLog: true
diff --git a/docker/gaffer-gremlin/conf/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store.properties
new file mode 100644
index 00000000..2195d783
--- /dev/null
+++ b/docker/gaffer-gremlin/conf/gaffer/store.properties
@@ -0,0 +1,20 @@
+#
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore
+# These can be configured to an existing graph deployment
+gaffer.host=localhost
+gaffer.port=8080
+gaffer.context-root=/rest/latest
diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
new file mode 100644
index 00000000..156805b0
--- /dev/null
+++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
@@ -0,0 +1,19 @@
+#
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
+gaffer.graphId=graphProxy
+gaffer.storeproperties=conf/gaffer/store.properties
+gaffer.userId=user01
diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml
new file mode 100644
index 00000000..46adb620
--- /dev/null
+++ b/docker/gaffer-gremlin/pom.xml
@@ -0,0 +1,52 @@
+
+
+
+ 4.0.0
+
+ uk.gov.gchq.gaffer.docker
+ gaffer-gremlin
+ 0.0.1-SNAPSHOT
+ pom
+
+
+ 3.7.1
+ 2.1.1-SNAPSHOT
+ 2.36
+
+
+
+
+
+ uk.gov.gchq.gaffer
+ tinkerpop
+ ${gaffer.version}
+
+
+
+ org.glassfish.jersey.inject
+ jersey-hk2
+ ${jersey.version}
+
+
+
+ org.apache.tinkerpop
+ gremlin-core
+ ${tinkerpop.version}
+
+
+
+
\ No newline at end of file
From 596d794f32a1a384a2966c71cda67fb36f2dd6c5 Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Fri, 8 Dec 2023 16:03:19 +0000
Subject: [PATCH 02/12] typo
---
docker/gaffer-gremlin/{README..md => README.md} | 0
docker/gaffer-gremlin/pom.xml | 2 +-
2 files changed, 1 insertion(+), 1 deletion(-)
rename docker/gaffer-gremlin/{README..md => README.md} (100%)
diff --git a/docker/gaffer-gremlin/README..md b/docker/gaffer-gremlin/README.md
similarity index 100%
rename from docker/gaffer-gremlin/README..md
rename to docker/gaffer-gremlin/README.md
diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml
index 46adb620..69957d2a 100644
--- a/docker/gaffer-gremlin/pom.xml
+++ b/docker/gaffer-gremlin/pom.xml
@@ -49,4 +49,4 @@
-
\ No newline at end of file
+
From 25ca55fee65fbc6c84567b181b68e48f3937ff7c Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 19 Dec 2023 14:02:56 +0000
Subject: [PATCH 03/12] tidy and hook into existing CI
---
cd/build_images.sh | 22 ++++++++++---------
docker/accumulo2.env | 7 +++---
docker/gaffer-gremlin/build.sh | 20 ++++++++++++-----
docker/gaffer-gremlin/pom.xml | 3 ++-
.../.env | 0
.../Dockerfile | 0
.../README.md | 0
.../gafferpop/gaffer/schema/elements.json | 0
.../conf/gafferpop/gaffer/schema/types.json | 0
.../conf/gafferpop/gaffer/store.properties | 0
.../gafferpop-tinkerpop-modern.properties | 0
.../conf/graphConfig.json | 0
.../conf/gremlin-server-empty-gaffer.yaml | 0
.../docker-compose.yaml | 0
.../files/.gitignore | 0
.../gremlin-gaffer-modern-example.ipynb | 0
16 files changed, 32 insertions(+), 20 deletions(-)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/.env (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/Dockerfile (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/README.md (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gaffer/schema/elements.json (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gaffer/schema/types.json (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gaffer/store.properties (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gafferpop-tinkerpop-modern.properties (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/graphConfig.json (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gremlin-server-empty-gaffer.yaml (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/docker-compose.yaml (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/files/.gitignore (100%)
rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/gremlin-gaffer-modern-example.ipynb (100%)
diff --git a/cd/build_images.sh b/cd/build_images.sh
index 9a8b4a08..681a94ed 100755
--- a/cd/build_images.sh
+++ b/cd/build_images.sh
@@ -18,22 +18,22 @@
# Required for variables from sourced env file to automatically be visible to docker compose.
set -e -a
-root_directory="$( cd $(dirname $(dirname $0)) > /dev/null 2>&1 && pwd )"
-cd $root_directory
+ROOT_DIR="$("$(dirname "$(dirname "${0}")")")"
+cd "${ROOT_DIR}"
-if [ ! -z "$1" ]; then
- ENV_FILE=$1
-else
- echo "Error - Environment file not set"; exit 1;
-fi
-
-# The following command sets:
+# The following env file will be sourced to set:
# HADOOP_VERSION
# GAFFER_VERSION
# GAFFERPY_VERSION
# ACCUMULO_VERSION
# SPARK_VERSION
-source "${ENV_FILE}"
+# TINKERPOP_VERSION
+if [[ -f "${1}" ]]; then
+ source "${1}"
+else
+ echo "Error - Environment file not set"
+ exit 1
+fi
# Builds all of the Gaffer and Accumulo related images:
docker compose --project-directory ./docker/accumulo/ -f ./docker/accumulo/docker-compose.yaml build
@@ -41,6 +41,8 @@ docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./doc
# Builds all of the notebook related images:
docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook
docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build
+# Builds the Gaffer Gremlin server
+./docker/gaffer-gremlin/build.sh
# Set $JHUB_OPTIONS_SERVER_VERSION
source ./docker/gaffer-jhub-options-server/get-version.sh
diff --git a/docker/accumulo2.env b/docker/accumulo2.env
index d4d48ddb..d3fa6384 100644
--- a/docker/accumulo2.env
+++ b/docker/accumulo2.env
@@ -3,7 +3,8 @@ HADOOP_VERSION=3.3.3
ACCUMULO_VERSION=2.0.1
HADOOP_CONF_DIR=/etc/hadoop/conf
ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
SPARK_VERSION=3.1.2
+TINKERPOP_VERSION=3.7.1
diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh
index 866b924f..2348a958 100755
--- a/docker/gaffer-gremlin/build.sh
+++ b/docker/gaffer-gremlin/build.sh
@@ -13,18 +13,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-declare CONTAINER_VER="2.1.0"
-declare CURRENT_DIR="$(dirname "${0}")"
+declare SCRIPT_DIR="$(dirname "${0}")"
-pushd "${CURRENT_DIR}"
+# Check if already have env var for version
+[[ -z "${GAFFER_VERSION}" ]] && \
+ echo "Missing GAFFER_VERSION env var" && \
+ exit 1
+[[ -z "${TINKERPOP_VERSION}" ]] && \
+ echo "Missing TINKERPOP_VERSION env var" && \
+ exit 1
+
+# Build from relevant directory
+pushd "${SCRIPT_DIR}" || exit 1
# Download JARs
- mvn clean dependency:copy-dependencies
+ mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}"
# Build container
- docker build -t gaffer-gremlin:"${CONTAINER_VER}" .
+ docker build -t gaffer-gremlin:"${GAFFER_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" .
# Clean
mvn clean
-popd
+popd || exit 1
echo "Build Successful"
diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml
index 69957d2a..cbd16a74 100644
--- a/docker/gaffer-gremlin/pom.xml
+++ b/docker/gaffer-gremlin/pom.xml
@@ -22,9 +22,10 @@
0.0.1-SNAPSHOT
pom
+
+ 2.1.0
3.7.1
- 2.1.1-SNAPSHOT
2.36
diff --git a/docker/gremlin-gaffer/.env b/docker/gremlin-gaffer-demo/.env
similarity index 100%
rename from docker/gremlin-gaffer/.env
rename to docker/gremlin-gaffer-demo/.env
diff --git a/docker/gremlin-gaffer/Dockerfile b/docker/gremlin-gaffer-demo/Dockerfile
similarity index 100%
rename from docker/gremlin-gaffer/Dockerfile
rename to docker/gremlin-gaffer-demo/Dockerfile
diff --git a/docker/gremlin-gaffer/README.md b/docker/gremlin-gaffer-demo/README.md
similarity index 100%
rename from docker/gremlin-gaffer/README.md
rename to docker/gremlin-gaffer-demo/README.md
diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json b/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json
similarity index 100%
rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json
rename to docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json
diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json b/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json
similarity index 100%
rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json
rename to docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json
diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties b/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties
similarity index 100%
rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties
rename to docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties
diff --git a/docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties b/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties
similarity index 100%
rename from docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties
rename to docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties
diff --git a/docker/gremlin-gaffer/conf/graphConfig.json b/docker/gremlin-gaffer-demo/conf/graphConfig.json
similarity index 100%
rename from docker/gremlin-gaffer/conf/graphConfig.json
rename to docker/gremlin-gaffer-demo/conf/graphConfig.json
diff --git a/docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml b/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml
similarity index 100%
rename from docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml
rename to docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml
diff --git a/docker/gremlin-gaffer/docker-compose.yaml b/docker/gremlin-gaffer-demo/docker-compose.yaml
similarity index 100%
rename from docker/gremlin-gaffer/docker-compose.yaml
rename to docker/gremlin-gaffer-demo/docker-compose.yaml
diff --git a/docker/gremlin-gaffer/files/.gitignore b/docker/gremlin-gaffer-demo/files/.gitignore
similarity index 100%
rename from docker/gremlin-gaffer/files/.gitignore
rename to docker/gremlin-gaffer-demo/files/.gitignore
diff --git a/docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb b/docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb
similarity index 100%
rename from docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb
rename to docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb
From d0dddcdff2173c8a94caa194fd713b0047bcaeea Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 19 Dec 2023 14:14:43 +0000
Subject: [PATCH 04/12] typo
---
.github/workflows/continuous-integration.yaml | 3 ++-
cd/build_images.sh | 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml
index d55dba4f..849f0290 100644
--- a/.github/workflows/continuous-integration.yaml
+++ b/.github/workflows/continuous-integration.yaml
@@ -20,6 +20,7 @@ on:
branches:
- 'release/**'
- 'hotfix/**'
+ workflow_dispatch:
jobs:
build:
@@ -55,7 +56,7 @@ jobs:
- name: Deploy to Kubernetes
run: ./cd/deploy_to_kind.sh
-
+
- name: Run gaffer-road-traffic Tests
run: helm test gaffer || (kubectl get po && kubectl describe po && kubectl logs -l app.kubernetes.io/component=test --tail=-1 && df -h && false)
diff --git a/cd/build_images.sh b/cd/build_images.sh
index 681a94ed..f319dfd8 100755
--- a/cd/build_images.sh
+++ b/cd/build_images.sh
@@ -29,10 +29,10 @@ cd "${ROOT_DIR}"
# SPARK_VERSION
# TINKERPOP_VERSION
if [[ -f "${1}" ]]; then
- source "${1}"
+ source "${1}"
else
- echo "Error - Environment file not set"
- exit 1
+ echo "Error - Environment file not set"
+ exit 1
fi
# Builds all of the Gaffer and Accumulo related images:
From 9725644673bda65f14498225f9587d78a575263c Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 19 Dec 2023 14:58:58 +0000
Subject: [PATCH 05/12] typo
---
.github/workflows/continuous-integration.yaml | 1 -
cd/build_images.sh | 7 +++++--
docker/gaffer-gremlin/build.sh | 2 +-
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml
index 849f0290..05775052 100644
--- a/.github/workflows/continuous-integration.yaml
+++ b/.github/workflows/continuous-integration.yaml
@@ -20,7 +20,6 @@ on:
branches:
- 'release/**'
- 'hotfix/**'
- workflow_dispatch:
jobs:
build:
diff --git a/cd/build_images.sh b/cd/build_images.sh
index f319dfd8..5d2c9f09 100755
--- a/cd/build_images.sh
+++ b/cd/build_images.sh
@@ -18,8 +18,9 @@
# Required for variables from sourced env file to automatically be visible to docker compose.
set -e -a
-ROOT_DIR="$("$(dirname "$(dirname "${0}")")")"
-cd "${ROOT_DIR}"
+ROOT_DIR="$(readlink -f "$(dirname "$(dirname "${0}")")")"
+
+pushd "${ROOT_DIR}" || exit 1
# The following env file will be sourced to set:
# HADOOP_VERSION
@@ -48,3 +49,5 @@ docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docke
source ./docker/gaffer-jhub-options-server/get-version.sh
# Builds the jhub options server:
docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build
+
+popd || exit 1
diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh
index 2348a958..34594223 100755
--- a/docker/gaffer-gremlin/build.sh
+++ b/docker/gaffer-gremlin/build.sh
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-declare SCRIPT_DIR="$(dirname "${0}")"
+declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")"
# Check if already have env var for version
[[ -z "${GAFFER_VERSION}" ]] && \
From 9f44a5a5ea2f749a4fb2b03f1a6ed5868887b998 Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 19 Dec 2023 15:08:38 +0000
Subject: [PATCH 06/12] readme updates
---
docker/gaffer-gremlin/README.md | 22 ++++++++++++++++------
1 file changed, 16 insertions(+), 6 deletions(-)
diff --git a/docker/gaffer-gremlin/README.md b/docker/gaffer-gremlin/README.md
index 7a1862bc..94d8803b 100644
--- a/docker/gaffer-gremlin/README.md
+++ b/docker/gaffer-gremlin/README.md
@@ -2,8 +2,8 @@
## Build
-Run the supplied `build.sh` script to pull all dependencies and build the
-container.
+Run the supplied `build.sh` script with the required environment vars to pull
+all dependencies and build the container.
_Note requires Maven and Docker_
@@ -12,17 +12,27 @@ Optionally run each command separately to configure the containers tags etc.
## Configure
The container will use the Gaffer Proxy store to connect to an existing graph
-and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/3.7.1/reference/#connecting-gremlin-server)).
+and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)).
The Gaffer graph the container will connect to can be configured as usual by
editing the `store.properties` file, which you can also bind mount over on an
existing image.
-The configuration for the Gremlin server is provided by the `[gaffer-gremlin-server.yaml](./conf/gaffer-gremlin-server.yaml)`
-this again can be modified as needed or bind mounted over.
+The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml`
+this again can be modified as needed or bind mounted over. Please see the
+[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/)
+for more information on configuring this image.
## Run
Simply run the container to publish the Gremlin server this can then be
connected to via the address specified in the server's yaml config to use
-Gremlin traversal.
+Gremlin traversal. The server can then be connected via the Gremlin
+console e.g. if using python
+
+```python
+from gremlin_python.process.anonymous_traversal_source import traversal
+
+g = traversal().withRemote(
+ DriverRemoteConnection('ws://localhost:8182/gremlin'))
+```
From adf564302fbfbc1cd52505c908a0244ab640610f Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 19 Dec 2023 16:22:34 +0000
Subject: [PATCH 07/12] add to publish ci
---
cd/publish_images.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/cd/publish_images.sh b/cd/publish_images.sh
index 86c26061..cd165b81 100755
--- a/cd/publish_images.sh
+++ b/cd/publish_images.sh
@@ -101,6 +101,7 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}"
pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}"
pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}"
+pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}"
pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}"
pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}"
pushContainer gchq/spark-py "${SPARK_VERSION}"
From 5e59949427c5d85d83c07a7174a3987663bd2625 Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 9 Jan 2024 16:01:55 +0000
Subject: [PATCH 08/12] merge existing demo image with new image
---
.gitignore | 1 +
docker/gaffer-gremlin/Dockerfile | 1 +
docker/gaffer-gremlin/README.md | 38 +++++++++----
docker/gaffer-gremlin/build.sh | 2 +-
.../conf/gaffer-gremlin-server.yaml | 8 ++-
.../conf}/gaffer/schema/elements.json | 0
.../conf}/gaffer/schema/types.json | 0
.../conf/gaffer/store-accumulo.properties} | 4 +-
.../conf/gafferpop/gafferpop.properties | 3 +-
.../example/compose.yaml} | 54 +++++--------------
.../gremlin-gaffer-modern-example.ipynb | 29 ++++++----
docker/gaffer-gremlin/pom.xml | 7 ---
docker/gremlin-gaffer-demo/.env | 7 ---
docker/gremlin-gaffer-demo/Dockerfile | 51 ------------------
docker/gremlin-gaffer-demo/README.md | 39 --------------
.../gafferpop-tinkerpop-modern.properties | 20 -------
.../gremlin-gaffer-demo/conf/graphConfig.json | 3 --
.../conf/gremlin-server-empty-gaffer.yaml | 43 ---------------
docker/gremlin-gaffer-demo/files/.gitignore | 2 -
19 files changed, 73 insertions(+), 239 deletions(-)
rename docker/{gremlin-gaffer-demo/conf/gafferpop => gaffer-gremlin/conf}/gaffer/schema/elements.json (100%)
rename docker/{gremlin-gaffer-demo/conf/gafferpop => gaffer-gremlin/conf}/gaffer/schema/types.json (100%)
rename docker/{gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties => gaffer-gremlin/conf/gaffer/store-accumulo.properties} (92%)
rename docker/{gremlin-gaffer-demo/docker-compose.yaml => gaffer-gremlin/example/compose.yaml} (76%)
rename docker/{gremlin-gaffer-demo => gaffer-gremlin/example}/gremlin-gaffer-modern-example.ipynb (86%)
delete mode 100644 docker/gremlin-gaffer-demo/.env
delete mode 100644 docker/gremlin-gaffer-demo/Dockerfile
delete mode 100644 docker/gremlin-gaffer-demo/README.md
delete mode 100644 docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties
delete mode 100644 docker/gremlin-gaffer-demo/conf/graphConfig.json
delete mode 100644 docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml
delete mode 100644 docker/gremlin-gaffer-demo/files/.gitignore
diff --git a/.gitignore b/.gitignore
index 597bd3b1..fa0d04b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,4 @@
**/.DS_Store
kubernetes/*/Chart.lock
kubernetes/*/charts/
+target/
diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile
index 30917f97..de1a5fa6 100644
--- a/docker/gaffer-gremlin/Dockerfile
+++ b/docker/gaffer-gremlin/Dockerfile
@@ -15,6 +15,7 @@
ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
ARG BASE_IMAGE_TAG=3.7.1
+# Base image provides WORKDIR, USER and ENTRYPOINT
FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
# Copy JARS
diff --git a/docker/gaffer-gremlin/README.md b/docker/gaffer-gremlin/README.md
index 94d8803b..0dfd36a0 100644
--- a/docker/gaffer-gremlin/README.md
+++ b/docker/gaffer-gremlin/README.md
@@ -2,8 +2,8 @@
## Build
-Run the supplied `build.sh` script with the required environment vars to pull
-all dependencies and build the container.
+Run the supplied `build.sh` script with the required environment vars (can
+source `accumolo2.env` file) to pull all dependencies and build the container.
_Note requires Maven and Docker_
@@ -11,12 +11,16 @@ Optionally run each command separately to configure the containers tags etc.
## Configure
-The container will use the Gaffer Proxy store to connect to an existing graph
-and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)).
+The container will use the Gaffer Proxy store by default to connect to an
+existing graph and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)).
The Gaffer graph the container will connect to can be configured as usual by
editing the `store.properties` file, which you can also bind mount over on an
-existing image.
+existing image. The config file locations are under the predefined workdir
+set by the parent gremlin server image the key locations in the image are:
+
+- `/opt/gremlin-server/conf/gaffer/store.properties` - Override for custom store properties.
+- `/opt/gremlin-server/conf/gafferpop/gafferpop.properties` - Override to configure the graph.
The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml`
this again can be modified as needed or bind mounted over. Please see the
@@ -25,14 +29,28 @@ for more information on configuring this image.
## Run
-Simply run the container to publish the Gremlin server this can then be
-connected to via the address specified in the server's yaml config to use
-Gremlin traversal. The server can then be connected via the Gremlin
-console e.g. if using python
+Simply run the container to publish the Gremlin server making sure to make the
+configured port available (port 8182 by default) this can then be connected to
+via the address specified in the server's yaml config to use Gremlin traversal.
+The server can then be connected via the Gremlin console e.g. if using
+`gremlinpython`:
```python
from gremlin_python.process.anonymous_traversal_source import traversal
g = traversal().withRemote(
- DriverRemoteConnection('ws://localhost:8182/gremlin'))
+ DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))
+```
+
+### Demo Deployment
+
+A demo/example using the tinkerpop 'modern' dataset and accumulo backed Gaffer
+is available under the `example` directory. This can be ran using docker compose
+to deploy the containers then, the provided jupiter notebook demonstrates how to
+connect and some basic queries on the data using `gremlinpython`.
+
+Run the example with:
+
+```bash
+docker compose up
```
diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh
index 34594223..e27a01dd 100755
--- a/docker/gaffer-gremlin/build.sh
+++ b/docker/gaffer-gremlin/build.sh
@@ -29,7 +29,7 @@ pushd "${SCRIPT_DIR}" || exit 1
mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}"
# Build container
- docker build -t gaffer-gremlin:"${GAFFER_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" .
+ docker build -t gchq/gaffer-gremlin:"${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" .
# Clean
mvn clean
diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
index 6dd17d7a..8d9aa562 100644
--- a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
+++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
@@ -29,10 +29,14 @@ scriptEngines: {
}
}
serializers:
- - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
+ # application/vnd.graphbinary-v1.0
+ - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 }
+ # application/vnd.graphbinary-v1.0-stringd
+ - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }}
metrics: {
- slf4jReporter: {enabled: true, interval: 180000}}
+ slf4jReporter: {enabled: true, interval: 180000}
+}
strictTransactionManagement: false
idleConnectionTimeout: 0
keepAliveInterval: 0
diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json b/docker/gaffer-gremlin/conf/gaffer/schema/elements.json
similarity index 100%
rename from docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json
rename to docker/gaffer-gremlin/conf/gaffer/schema/elements.json
diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json b/docker/gaffer-gremlin/conf/gaffer/schema/types.json
similarity index 100%
rename from docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json
rename to docker/gaffer-gremlin/conf/gaffer/schema/types.json
diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties
similarity index 92%
rename from docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties
rename to docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties
index efb5b466..b82e1a7a 100644
--- a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties
+++ b/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties
@@ -1,5 +1,5 @@
#
-# Copyright 2016-2023 Crown Copyright
+# Copyright 2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -21,4 +21,4 @@ accumulo.user=root
accumulo.password=secret
# General store config
gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService
-gaffer.store.job.tracker.enabled=true
+gaffer.store.job.tracker.enabled=true
\ No newline at end of file
diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
index 156805b0..10e81747 100644
--- a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
+++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
@@ -14,6 +14,7 @@
# limitations under the License.
#
gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
-gaffer.graphId=graphProxy
+gaffer.graphId=graph1
gaffer.storeproperties=conf/gaffer/store.properties
+gaffer.schemas=conf/gaffer/schema/
gaffer.userId=user01
diff --git a/docker/gremlin-gaffer-demo/docker-compose.yaml b/docker/gaffer-gremlin/example/compose.yaml
similarity index 76%
rename from docker/gremlin-gaffer-demo/docker-compose.yaml
rename to docker/gaffer-gremlin/example/compose.yaml
index 4bb32592..c2a392dd 100644
--- a/docker/gremlin-gaffer-demo/docker-compose.yaml
+++ b/docker/gaffer-gremlin/example/compose.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -42,10 +42,6 @@ services:
interval: 30s
timeout: 10s
retries: 3
- build:
- context: ../hdfs/
- args:
- HADOOP_VERSION: ${HADOOP_VERSION}
command: namenode
container_name: hdfs-namenode
hostname: hdfs-namenode
@@ -54,7 +50,7 @@ services:
ports:
- 9870:9870
volumes:
- - ../hdfs/conf:${HADOOP_CONF_DIR}:ro
+ - ../../hdfs/conf:${HADOOP_CONF_DIR}:ro
- /var/log/hadoop
- /data1
- /data2
@@ -70,7 +66,7 @@ services:
environment:
- HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
volumes:
- - ../hdfs/conf:${HADOOP_CONF_DIR}:ro
+ - ../../hdfs/conf:${HADOOP_CONF_DIR}:ro
- /var/log/hadoop
- /data1
- /data2
@@ -86,12 +82,6 @@ services:
timeout: 5s
retries: 3
start_period: 10s
- build:
- context: .
- args:
- GAFFER_VERSION: ${GAFFER_VERSION}
- BASE_IMAGE_NAME: gchq/accumulo
- BASE_IMAGE_TAG: ${ACCUMULO_VERSION}
command: master
container_name: accumulo-master
hostname: accumulo-master
@@ -103,7 +93,7 @@ services:
# accumulo to "auth" with HDFS as the super user so that it can:
- HADOOP_USER_NAME=hadoop
volumes:
- - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
+ - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
- /var/log/accumulo
accumulo-tserver:
@@ -127,7 +117,7 @@ services:
# accumulo to "auth" with HDFS as the super user so that it can:
- HADOOP_USER_NAME=hadoop
volumes:
- - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
+ - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
- /var/log/accumulo
accumulo-monitor:
@@ -148,7 +138,7 @@ services:
ports:
- 9995:9995
volumes:
- - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
+ - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
- /var/log/accumulo
accumulo-gc:
@@ -167,37 +157,17 @@ services:
# accumulo to "auth" with HDFS as the super user so that it can:
- HADOOP_USER_NAME=hadoop
volumes:
- - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
+ - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro
- /var/log/accumulo
- gaffer-rest:
- image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}
+ gaffer-gremlin:
+ image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}
depends_on:
accumulo-tserver:
condition: service_healthy
- build:
- context: ../gaffer-rest/
- args:
- GAFFER_VERSION: ${GAFFER_VERSION}
- ACCUMULO_VERSION: ${ACCUMULO_VERSION}
- ports:
- - 8080:8080
- volumes:
- - ./conf/graphConfig.json:/gaffer/graph/graphConfig.json:ro
- - ./conf/gafferpop/gaffer/store.properties:/gaffer/store/store.properties:ro
- - ./conf/gafferpop/gaffer/schema:/gaffer/schema:ro
-
- gremlin-gaffer:
- image: gchq/gremlin-gaffer:${GREMLIN_VERSION}-gaffer-${GAFFER_VERSION}
- depends_on:
- accumulo-tserver:
- condition: service_healthy
- build:
- context: .
- args:
- GAFFER_VERSION: ${GAFFER_VERSION}
- GREMLIN_VERSION: ${GREMLIN_VERSION}
ports:
- 8182:8182
volumes:
- - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro
+ - ../conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro
+ - ../conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro
+ - ../conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro
diff --git a/docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb b/docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb
similarity index 86%
rename from docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb
rename to docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb
index d0a6bb79..00d5643c 100644
--- a/docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb
+++ b/docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb
@@ -10,7 +10,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
@@ -30,9 +30,20 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 24,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"# Create gremlin client to run gremlin scripts directly\n",
"client = Client('ws://localhost:8182/gremlin', 'g')\n",
@@ -42,7 +53,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
@@ -52,7 +63,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 26,
"metadata": {},
"outputs": [
{
@@ -70,7 +81,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 27,
"metadata": {},
"outputs": [
{
@@ -79,7 +90,7 @@
"[0.5]"
]
},
- "execution_count": 15,
+ "execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
@@ -91,7 +102,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 28,
"metadata": {},
"outputs": [
{
@@ -100,7 +111,7 @@
"[path[v[1], v[3]], path[v[1], v[4], v[3]]]"
]
},
- "execution_count": 16,
+ "execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml
index cbd16a74..61ab0432 100644
--- a/docker/gaffer-gremlin/pom.xml
+++ b/docker/gaffer-gremlin/pom.xml
@@ -25,7 +25,6 @@
2.1.0
- 3.7.1
2.36
@@ -42,12 +41,6 @@
jersey-hk2
${jersey.version}
-
-
- org.apache.tinkerpop
- gremlin-core
- ${tinkerpop.version}
-
diff --git a/docker/gremlin-gaffer-demo/.env b/docker/gremlin-gaffer-demo/.env
deleted file mode 100644
index b9a28848..00000000
--- a/docker/gremlin-gaffer-demo/.env
+++ /dev/null
@@ -1,7 +0,0 @@
-ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GREMLIN_VERSION=3.6.4
-ACCUMULO_VERSION=2.0.1
-HADOOP_VERSION=3.3.3
-ACCUMULO_CONF_DIR=/etc/accumulo/conf
-HADOOP_CONF_DIR=/etc/hadoop/conf
diff --git a/docker/gremlin-gaffer-demo/Dockerfile b/docker/gremlin-gaffer-demo/Dockerfile
deleted file mode 100644
index ffb80399..00000000
--- a/docker/gremlin-gaffer-demo/Dockerfile
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2023 Crown Copyright
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG BUILDER_IMAGE_NAME=maven
-ARG BUILDER_IMAGE_TAG=3.8.4-jdk-8
-
-ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server
-ARG BASE_IMAGE_TAG=3.6.4
-
-FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
-
-ARG BASE_IMAGE_TAG
-ARG GAFFER_VERSION=2.0.0
-ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
-ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
-
-WORKDIR /jars
-
-# Allow users to provide their own JAR files
-COPY ./files/ .
-# Try to download required version from Maven Central, otherwise build from source
-RUN allFilesDownloaded="TRUE" && \
- if [ ! -f "./tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ] && [ "${allFilesDownloaded}" = "TRUE" ]; then \
- wget -nv "${GAFFER_DOWNLOAD_URL}/uk/gov/gchq/gaffer/tinkerpop/${GAFFER_VERSION}/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" || allFilesDownloaded="FALSE"; \
- fi && \
- if [ "${allFilesDownloaded}" = "FALSE" ]; then \
- git clone ${GAFFER_GIT_REPO} /tmp/gaffer && \
- cd /tmp/gaffer && \
- git checkout ${GAFFER_VERSION} || git checkout gaffer2-${GAFFER_VERSION} && \
- mvn clean package -Pquick -pl :tinkerpop && \
- if [ ! -f "/jars/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ]; then \
- cp ./library/tinkerpop/target/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar /jars; \
- fi \
- fi
-
-FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
-COPY --from=builder --chown=root:root /jars/*.jar ext/gafferpop/plugin/
-COPY ./conf/gafferpop/ conf/gafferpop/
-COPY ./conf/gremlin-server-empty-gaffer.yaml conf/gremlin-server-empty-gaffer.yaml
-CMD ["conf/gremlin-server-empty-gaffer.yaml"]
diff --git a/docker/gremlin-gaffer-demo/README.md b/docker/gremlin-gaffer-demo/README.md
deleted file mode 100644
index 018e76d6..00000000
--- a/docker/gremlin-gaffer-demo/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-Gremlin Gaffer Plugin
-======
-In this folder you can find the required files for building and running a gremlin-server with the Gaffer plugin loaded.
-
-The Docker image uses TinkerPop's gremlin-server with GafferPop config and plugin jars added in.
-When run with docker compose it will provide you a full accumulo ecosystem complete with [hdfs](../hdfs) and the [Gaffer REST API](../gaffer-rest).
-
-# Running Locally
-The easiest way to build and run these services is to use docker compose, by running the following from this directory:
-```bash
-docker compose up
-```
-
-## Example Notebook
-See `gremlin-gaffer-modern-example.ipynb` for an example using the "TinkerPop Modern" demo graph.
-
-## Customising the build
-Custom Gaffer TinkerPop plugin jars can be added in the files directory. The Gaffer schema, store properties and gafferpop properties can be found in `conf/gafferpop` and are customised in a docker compose build using volumes. The `gremlin-server-empty-gaffer.yaml` cannot be overwritten in a volume, it must be built into the image.
-
-## Containers that are started:
-* Zookeeper
-* HDFS
- * Datanode
- * Namenode
-* Accumulo
- * Monitor
- * GC
- * tserver
- * Master
-* Gaffer REST
-* Gremlin Server with GafferPop
-
-Access the HDFS NameNode web UI at: http://localhost:9870
-
-Access the Accumulo Monitor UI at: http://localhost:9995
-
-Access the Gaffer REST API at: http://localhost:8080/rest/
-
-Access the Gremlin Server with GafferPop at: http://localhost:8182/
diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties b/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties
deleted file mode 100644
index 2bb1fbfb..00000000
--- a/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Copyright 2016-2023 Crown Copyright
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
-gaffer.graphId=graph1
-gaffer.storeproperties=conf/gafferpop/gaffer/store.properties
-gaffer.schemas=conf/gafferpop/gaffer/schema/
-gaffer.userId=user01
diff --git a/docker/gremlin-gaffer-demo/conf/graphConfig.json b/docker/gremlin-gaffer-demo/conf/graphConfig.json
deleted file mode 100644
index 57f97103..00000000
--- a/docker/gremlin-gaffer-demo/conf/graphConfig.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "graphId": "graph1"
-}
\ No newline at end of file
diff --git a/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml b/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml
deleted file mode 100644
index 0fae7ec1..00000000
--- a/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright 2023 Crown Copyright
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-host: localhost
-port: 8182
-evaluationTimeout: 30000
-graphs: {
- graph: conf/gafferpop/gafferpop-tinkerpop-modern.properties}
-scriptEngines: {
- gremlin-groovy: {
- plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {},
- uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {},
- org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true},
- org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},
- org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}}
- }
-}
-serializers:
- - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} # application/vnd.graphbinary-v1.0-stringd
-
-metrics: {
- slf4jReporter: {enabled: true, interval: 180000}}
-strictTransactionManagement: false
-idleConnectionTimeout: 0
-keepAliveInterval: 0
-maxInitialLineLength: 4096
-maxHeaderSize: 8192
-maxChunkSize: 8192
-maxContentLength: 10485760
-maxAccumulationBufferComponents: 1024
-resultIterationBatchSize: 64
-enableAuditLog: true
diff --git a/docker/gremlin-gaffer-demo/files/.gitignore b/docker/gremlin-gaffer-demo/files/.gitignore
deleted file mode 100644
index fd807ac5..00000000
--- a/docker/gremlin-gaffer-demo/files/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*.tar.gz
-*.jar
From c333d6545a2189d409a5a53eed71ba8dfc5f7899 Mon Sep 17 00:00:00 2001
From: tb06904 <141412860+tb06904@users.noreply.github.com>
Date: Tue, 9 Jan 2024 16:47:30 +0000
Subject: [PATCH 09/12] move build to docker compose and tidy
---
docker/gaffer-gremlin/Dockerfile | 2 +-
docker/gaffer-gremlin/build.sh | 4 +--
docker/gaffer-gremlin/compose.yaml | 25 +++++++++++++++++++
.../conf/gaffer-gremlin-server.yaml | 2 +-
.../conf/gaffer/store.properties | 2 +-
.../conf/gafferpop/gafferpop.properties | 2 +-
docker/gaffer-gremlin/example/compose.yaml | 6 ++---
.../conf/gaffer/schema/elements.json | 0
.../conf/gaffer/schema/types.json | 0
.../conf/gaffer/store-accumulo.properties | 2 +-
.../conf/gafferpop/gafferpop.properties | 20 +++++++++++++++
docker/gaffer-gremlin/pom.xml | 2 +-
12 files changed, 56 insertions(+), 11 deletions(-)
create mode 100644 docker/gaffer-gremlin/compose.yaml
rename docker/gaffer-gremlin/{ => example}/conf/gaffer/schema/elements.json (100%)
rename docker/gaffer-gremlin/{ => example}/conf/gaffer/schema/types.json (100%)
rename docker/gaffer-gremlin/{ => example}/conf/gaffer/store-accumulo.properties (96%)
create mode 100644 docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties
diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile
index de1a5fa6..5fa93c14 100644
--- a/docker/gaffer-gremlin/Dockerfile
+++ b/docker/gaffer-gremlin/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh
index e27a01dd..a35b6872 100755
--- a/docker/gaffer-gremlin/build.sh
+++ b/docker/gaffer-gremlin/build.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -29,7 +29,7 @@ pushd "${SCRIPT_DIR}" || exit 1
mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}"
# Build container
- docker build -t gchq/gaffer-gremlin:"${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" .
+ docker compose build
# Clean
mvn clean
diff --git a/docker/gaffer-gremlin/compose.yaml b/docker/gaffer-gremlin/compose.yaml
new file mode 100644
index 00000000..ad8370f6
--- /dev/null
+++ b/docker/gaffer-gremlin/compose.yaml
@@ -0,0 +1,25 @@
+# Copyright 2024 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.7"
+
+services:
+ gaffer-gremlin:
+ image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}
+ build:
+ context: .
+ args:
+ BASE_IMAGE_TAG: ${TINKERPOP_VERSION}
+ ports:
+ - 8182:8182
diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
index 8d9aa562..acebb348 100644
--- a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
+++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docker/gaffer-gremlin/conf/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store.properties
index 2195d783..6dad3cd4 100644
--- a/docker/gaffer-gremlin/conf/gaffer/store.properties
+++ b/docker/gaffer-gremlin/conf/gaffer/store.properties
@@ -1,5 +1,5 @@
#
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
index 10e81747..e4e662b0 100644
--- a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
+++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties
@@ -1,5 +1,5 @@
#
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docker/gaffer-gremlin/example/compose.yaml b/docker/gaffer-gremlin/example/compose.yaml
index c2a392dd..51c34e44 100644
--- a/docker/gaffer-gremlin/example/compose.yaml
+++ b/docker/gaffer-gremlin/example/compose.yaml
@@ -168,6 +168,6 @@ services:
ports:
- 8182:8182
volumes:
- - ../conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro
- - ../conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro
- - ../conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro
+ - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro
+ - ./conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro
+ - ./conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro
diff --git a/docker/gaffer-gremlin/conf/gaffer/schema/elements.json b/docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json
similarity index 100%
rename from docker/gaffer-gremlin/conf/gaffer/schema/elements.json
rename to docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json
diff --git a/docker/gaffer-gremlin/conf/gaffer/schema/types.json b/docker/gaffer-gremlin/example/conf/gaffer/schema/types.json
similarity index 100%
rename from docker/gaffer-gremlin/conf/gaffer/schema/types.json
rename to docker/gaffer-gremlin/example/conf/gaffer/schema/types.json
diff --git a/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties b/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties
similarity index 96%
rename from docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties
rename to docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties
index b82e1a7a..212615d9 100644
--- a/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties
+++ b/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties
@@ -1,5 +1,5 @@
#
-# Copyright 2024 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties
new file mode 100644
index 00000000..e4e662b0
--- /dev/null
+++ b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties
@@ -0,0 +1,20 @@
+#
+# Copyright 2023-2024 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph
+gaffer.graphId=graph1
+gaffer.storeproperties=conf/gaffer/store.properties
+gaffer.schemas=conf/gaffer/schema/
+gaffer.userId=user01
diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml
index 61ab0432..65f525f3 100644
--- a/docker/gaffer-gremlin/pom.xml
+++ b/docker/gaffer-gremlin/pom.xml
@@ -1,6 +1,6 @@