From 626f4237a1652b2ebe7dbd180a88bd63d3ea1a37 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:01:31 +0000 Subject: [PATCH 01/12] add basic proof of concept using proxy store --- docker/gaffer-gremlin/Dockerfile | 27 ++++++++++ docker/gaffer-gremlin/README..md | 28 ++++++++++ docker/gaffer-gremlin/build.sh | 30 +++++++++++ .../conf/gaffer-gremlin-server.yaml | 45 ++++++++++++++++ .../conf/gaffer/store.properties | 20 +++++++ .../conf/gafferpop/gafferpop.properties | 19 +++++++ docker/gaffer-gremlin/pom.xml | 52 +++++++++++++++++++ 7 files changed, 221 insertions(+) create mode 100644 docker/gaffer-gremlin/Dockerfile create mode 100644 docker/gaffer-gremlin/README..md create mode 100755 docker/gaffer-gremlin/build.sh create mode 100644 docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml create mode 100644 docker/gaffer-gremlin/conf/gaffer/store.properties create mode 100644 docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties create mode 100644 docker/gaffer-gremlin/pom.xml diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile new file mode 100644 index 00000000..30917f97 --- /dev/null +++ b/docker/gaffer-gremlin/Dockerfile @@ -0,0 +1,27 @@ +# Copyright 2023 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server +ARG BASE_IMAGE_TAG=3.7.1 + +FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} + +# Copy JARS +COPY ./target/dependency/*.jar ext/gafferpop/plugin/ + +# Copy configs +COPY ./conf/ conf/ + +# Set server to run +CMD ["conf/gaffer-gremlin-server.yaml"] diff --git a/docker/gaffer-gremlin/README..md b/docker/gaffer-gremlin/README..md new file mode 100644 index 00000000..7a1862bc --- /dev/null +++ b/docker/gaffer-gremlin/README..md @@ -0,0 +1,28 @@ +# gaffer-gremlin + +## Build + +Run the supplied `build.sh` script to pull all dependencies and build the +container. + +_Note requires Maven and Docker_ + +Optionally run each command separately to configure the containers tags etc. + +## Configure + +The container will use the Gaffer Proxy store to connect to an existing graph +and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/3.7.1/reference/#connecting-gremlin-server)). + +The Gaffer graph the container will connect to can be configured as usual by +editing the `store.properties` file, which you can also bind mount over on an +existing image. + +The configuration for the Gremlin server is provided by the `[gaffer-gremlin-server.yaml](./conf/gaffer-gremlin-server.yaml)` +this again can be modified as needed or bind mounted over. + +## Run + +Simply run the container to publish the Gremlin server this can then be +connected to via the address specified in the server's yaml config to use +Gremlin traversal. diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh new file mode 100755 index 00000000..866b924f --- /dev/null +++ b/docker/gaffer-gremlin/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2023 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare CONTAINER_VER="2.1.0" +declare CURRENT_DIR="$(dirname "${0}")" + +pushd "${CURRENT_DIR}" + # Download JARs + mvn clean dependency:copy-dependencies + + # Build container + docker build -t gaffer-gremlin:"${CONTAINER_VER}" . + + # Clean + mvn clean +popd + +echo "Build Successful" diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml new file mode 100644 index 00000000..6dd17d7a --- /dev/null +++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml @@ -0,0 +1,45 @@ +# Copyright 2023 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +host: localhost +port: 8182 +evaluationTimeout: 30000 +graphs: { + graph: conf/gafferpop/gafferpop.properties +} +scriptEngines: { + gremlin-groovy: { + plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]}, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]} + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} + +metrics: { + slf4jReporter: {enabled: true, interval: 180000}} +strictTransactionManagement: false +idleConnectionTimeout: 0 +keepAliveInterval: 0 +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 10485760 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +enableAuditLog: true diff --git a/docker/gaffer-gremlin/conf/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store.properties new file mode 100644 index 00000000..2195d783 --- /dev/null +++ b/docker/gaffer-gremlin/conf/gaffer/store.properties @@ -0,0 +1,20 @@ +# +# Copyright 2023 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore +# These can be configured to an existing graph deployment +gaffer.host=localhost +gaffer.port=8080 +gaffer.context-root=/rest/latest diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties new file mode 100644 index 00000000..156805b0 --- /dev/null +++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties @@ -0,0 +1,19 @@ +# +# Copyright 2023 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph +gaffer.graphId=graphProxy +gaffer.storeproperties=conf/gaffer/store.properties +gaffer.userId=user01 diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml new file mode 100644 index 00000000..46adb620 --- /dev/null +++ b/docker/gaffer-gremlin/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + uk.gov.gchq.gaffer.docker + gaffer-gremlin + 0.0.1-SNAPSHOT + pom + + + 3.7.1 + 2.1.1-SNAPSHOT + 2.36 + + + + + + uk.gov.gchq.gaffer + tinkerpop + ${gaffer.version} + + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} + + + + org.apache.tinkerpop + gremlin-core + ${tinkerpop.version} + + + + \ No newline at end of file From 596d794f32a1a384a2966c71cda67fb36f2dd6c5 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:03:19 +0000 Subject: [PATCH 02/12] typo --- docker/gaffer-gremlin/{README..md => README.md} | 0 docker/gaffer-gremlin/pom.xml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docker/gaffer-gremlin/{README..md => README.md} (100%) diff --git a/docker/gaffer-gremlin/README..md b/docker/gaffer-gremlin/README.md similarity index 100% rename from docker/gaffer-gremlin/README..md rename to docker/gaffer-gremlin/README.md diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml index 46adb620..69957d2a 100644 --- a/docker/gaffer-gremlin/pom.xml +++ b/docker/gaffer-gremlin/pom.xml @@ -49,4 +49,4 @@ - \ No newline at end of file + From 25ca55fee65fbc6c84567b181b68e48f3937ff7c Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:02:56 +0000 Subject: [PATCH 03/12] tidy and hook into existing CI --- cd/build_images.sh | 22 ++++++++++--------- docker/accumulo2.env | 7 +++--- docker/gaffer-gremlin/build.sh | 20 ++++++++++++----- docker/gaffer-gremlin/pom.xml | 3 ++- .../.env | 0 .../Dockerfile | 0 .../README.md | 0 .../gafferpop/gaffer/schema/elements.json | 0 .../conf/gafferpop/gaffer/schema/types.json | 0 .../conf/gafferpop/gaffer/store.properties | 0 .../gafferpop-tinkerpop-modern.properties | 0 .../conf/graphConfig.json | 0 .../conf/gremlin-server-empty-gaffer.yaml | 0 .../docker-compose.yaml | 0 .../files/.gitignore | 0 .../gremlin-gaffer-modern-example.ipynb | 0 16 files changed, 32 insertions(+), 20 deletions(-) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/.env (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/Dockerfile (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/README.md (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gaffer/schema/elements.json (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gaffer/schema/types.json (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gaffer/store.properties (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gafferpop/gafferpop-tinkerpop-modern.properties (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/graphConfig.json (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/conf/gremlin-server-empty-gaffer.yaml (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/docker-compose.yaml (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/files/.gitignore (100%) rename docker/{gremlin-gaffer => gremlin-gaffer-demo}/gremlin-gaffer-modern-example.ipynb (100%) diff --git a/cd/build_images.sh b/cd/build_images.sh index 9a8b4a08..681a94ed 100755 --- a/cd/build_images.sh +++ b/cd/build_images.sh @@ -18,22 +18,22 @@ # Required for variables from sourced env file to automatically be visible to docker compose. set -e -a -root_directory="$( cd $(dirname $(dirname $0)) > /dev/null 2>&1 && pwd )" -cd $root_directory +ROOT_DIR="$("$(dirname "$(dirname "${0}")")")" +cd "${ROOT_DIR}" -if [ ! -z "$1" ]; then - ENV_FILE=$1 -else - echo "Error - Environment file not set"; exit 1; -fi - -# The following command sets: +# The following env file will be sourced to set: # HADOOP_VERSION # GAFFER_VERSION # GAFFERPY_VERSION # ACCUMULO_VERSION # SPARK_VERSION -source "${ENV_FILE}" +# TINKERPOP_VERSION +if [[ -f "${1}" ]]; then + source "${1}" +else + echo "Error - Environment file not set" + exit 1 +fi # Builds all of the Gaffer and Accumulo related images: docker compose --project-directory ./docker/accumulo/ -f ./docker/accumulo/docker-compose.yaml build @@ -41,6 +41,8 @@ docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./doc # Builds all of the notebook related images: docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build +# Builds the Gaffer Gremlin server +./docker/gaffer-gremlin/build.sh # Set $JHUB_OPTIONS_SERVER_VERSION source ./docker/gaffer-jhub-options-server/get-version.sh diff --git a/docker/accumulo2.env b/docker/accumulo2.env index d4d48ddb..d3fa6384 100644 --- a/docker/accumulo2.env +++ b/docker/accumulo2.env @@ -3,7 +3,8 @@ HADOOP_VERSION=3.3.3 ACCUMULO_VERSION=2.0.1 HADOOP_CONF_DIR=/etc/hadoop/conf ACCUMULO_CONF_DIR=/etc/accumulo/conf -GAFFER_VERSION=2.0.0 -GAFFER_TESTER_VERSION=2.0.0 -GAFFERPY_VERSION=2.0.0 +GAFFER_VERSION=2.1.0 +GAFFER_TESTER_VERSION=2.1.0 +GAFFERPY_VERSION=2.1.0 SPARK_VERSION=3.1.2 +TINKERPOP_VERSION=3.7.1 diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh index 866b924f..2348a958 100755 --- a/docker/gaffer-gremlin/build.sh +++ b/docker/gaffer-gremlin/build.sh @@ -13,18 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -declare CONTAINER_VER="2.1.0" -declare CURRENT_DIR="$(dirname "${0}")" +declare SCRIPT_DIR="$(dirname "${0}")" -pushd "${CURRENT_DIR}" +# Check if already have env var for version +[[ -z "${GAFFER_VERSION}" ]] && \ + echo "Missing GAFFER_VERSION env var" && \ + exit 1 +[[ -z "${TINKERPOP_VERSION}" ]] && \ + echo "Missing TINKERPOP_VERSION env var" && \ + exit 1 + +# Build from relevant directory +pushd "${SCRIPT_DIR}" || exit 1 # Download JARs - mvn clean dependency:copy-dependencies + mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}" # Build container - docker build -t gaffer-gremlin:"${CONTAINER_VER}" . + docker build -t gaffer-gremlin:"${GAFFER_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" . # Clean mvn clean -popd +popd || exit 1 echo "Build Successful" diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml index 69957d2a..cbd16a74 100644 --- a/docker/gaffer-gremlin/pom.xml +++ b/docker/gaffer-gremlin/pom.xml @@ -22,9 +22,10 @@ 0.0.1-SNAPSHOT pom + + 2.1.0 3.7.1 - 2.1.1-SNAPSHOT 2.36 diff --git a/docker/gremlin-gaffer/.env b/docker/gremlin-gaffer-demo/.env similarity index 100% rename from docker/gremlin-gaffer/.env rename to docker/gremlin-gaffer-demo/.env diff --git a/docker/gremlin-gaffer/Dockerfile b/docker/gremlin-gaffer-demo/Dockerfile similarity index 100% rename from docker/gremlin-gaffer/Dockerfile rename to docker/gremlin-gaffer-demo/Dockerfile diff --git a/docker/gremlin-gaffer/README.md b/docker/gremlin-gaffer-demo/README.md similarity index 100% rename from docker/gremlin-gaffer/README.md rename to docker/gremlin-gaffer-demo/README.md diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json b/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json similarity index 100% rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json rename to docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json b/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json similarity index 100% rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json rename to docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties b/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties similarity index 100% rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties rename to docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties diff --git a/docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties b/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties similarity index 100% rename from docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties rename to docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties diff --git a/docker/gremlin-gaffer/conf/graphConfig.json b/docker/gremlin-gaffer-demo/conf/graphConfig.json similarity index 100% rename from docker/gremlin-gaffer/conf/graphConfig.json rename to docker/gremlin-gaffer-demo/conf/graphConfig.json diff --git a/docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml b/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml similarity index 100% rename from docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml rename to docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml diff --git a/docker/gremlin-gaffer/docker-compose.yaml b/docker/gremlin-gaffer-demo/docker-compose.yaml similarity index 100% rename from docker/gremlin-gaffer/docker-compose.yaml rename to docker/gremlin-gaffer-demo/docker-compose.yaml diff --git a/docker/gremlin-gaffer/files/.gitignore b/docker/gremlin-gaffer-demo/files/.gitignore similarity index 100% rename from docker/gremlin-gaffer/files/.gitignore rename to docker/gremlin-gaffer-demo/files/.gitignore diff --git a/docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb b/docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb similarity index 100% rename from docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb rename to docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb From d0dddcdff2173c8a94caa194fd713b0047bcaeea Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:14:43 +0000 Subject: [PATCH 04/12] typo --- .github/workflows/continuous-integration.yaml | 3 ++- cd/build_images.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index d55dba4f..849f0290 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -20,6 +20,7 @@ on: branches: - 'release/**' - 'hotfix/**' + workflow_dispatch: jobs: build: @@ -55,7 +56,7 @@ jobs: - name: Deploy to Kubernetes run: ./cd/deploy_to_kind.sh - + - name: Run gaffer-road-traffic Tests run: helm test gaffer || (kubectl get po && kubectl describe po && kubectl logs -l app.kubernetes.io/component=test --tail=-1 && df -h && false) diff --git a/cd/build_images.sh b/cd/build_images.sh index 681a94ed..f319dfd8 100755 --- a/cd/build_images.sh +++ b/cd/build_images.sh @@ -29,10 +29,10 @@ cd "${ROOT_DIR}" # SPARK_VERSION # TINKERPOP_VERSION if [[ -f "${1}" ]]; then - source "${1}" + source "${1}" else - echo "Error - Environment file not set" - exit 1 + echo "Error - Environment file not set" + exit 1 fi # Builds all of the Gaffer and Accumulo related images: From 9725644673bda65f14498225f9587d78a575263c Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:58:58 +0000 Subject: [PATCH 05/12] typo --- .github/workflows/continuous-integration.yaml | 1 - cd/build_images.sh | 7 +++++-- docker/gaffer-gremlin/build.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 849f0290..05775052 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -20,7 +20,6 @@ on: branches: - 'release/**' - 'hotfix/**' - workflow_dispatch: jobs: build: diff --git a/cd/build_images.sh b/cd/build_images.sh index f319dfd8..5d2c9f09 100755 --- a/cd/build_images.sh +++ b/cd/build_images.sh @@ -18,8 +18,9 @@ # Required for variables from sourced env file to automatically be visible to docker compose. set -e -a -ROOT_DIR="$("$(dirname "$(dirname "${0}")")")" -cd "${ROOT_DIR}" +ROOT_DIR="$(readlink -f "$(dirname "$(dirname "${0}")")")" + +pushd "${ROOT_DIR}" || exit 1 # The following env file will be sourced to set: # HADOOP_VERSION @@ -48,3 +49,5 @@ docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docke source ./docker/gaffer-jhub-options-server/get-version.sh # Builds the jhub options server: docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build + +popd || exit 1 diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh index 2348a958..34594223 100755 --- a/docker/gaffer-gremlin/build.sh +++ b/docker/gaffer-gremlin/build.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -declare SCRIPT_DIR="$(dirname "${0}")" +declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")" # Check if already have env var for version [[ -z "${GAFFER_VERSION}" ]] && \ From 9f44a5a5ea2f749a4fb2b03f1a6ed5868887b998 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:08:38 +0000 Subject: [PATCH 06/12] readme updates --- docker/gaffer-gremlin/README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docker/gaffer-gremlin/README.md b/docker/gaffer-gremlin/README.md index 7a1862bc..94d8803b 100644 --- a/docker/gaffer-gremlin/README.md +++ b/docker/gaffer-gremlin/README.md @@ -2,8 +2,8 @@ ## Build -Run the supplied `build.sh` script to pull all dependencies and build the -container. +Run the supplied `build.sh` script with the required environment vars to pull +all dependencies and build the container. _Note requires Maven and Docker_ @@ -12,17 +12,27 @@ Optionally run each command separately to configure the containers tags etc. ## Configure The container will use the Gaffer Proxy store to connect to an existing graph -and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/3.7.1/reference/#connecting-gremlin-server)). +and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)). The Gaffer graph the container will connect to can be configured as usual by editing the `store.properties` file, which you can also bind mount over on an existing image. -The configuration for the Gremlin server is provided by the `[gaffer-gremlin-server.yaml](./conf/gaffer-gremlin-server.yaml)` -this again can be modified as needed or bind mounted over. +The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml` +this again can be modified as needed or bind mounted over. Please see the +[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/) +for more information on configuring this image. ## Run Simply run the container to publish the Gremlin server this can then be connected to via the address specified in the server's yaml config to use -Gremlin traversal. +Gremlin traversal. The server can then be connected via the Gremlin +console e.g. if using python + +```python +from gremlin_python.process.anonymous_traversal_source import traversal + +g = traversal().withRemote( + DriverRemoteConnection('ws://localhost:8182/gremlin')) +``` From adf564302fbfbc1cd52505c908a0244ab640610f Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:22:34 +0000 Subject: [PATCH 07/12] add to publish ci --- cd/publish_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cd/publish_images.sh b/cd/publish_images.sh index 86c26061..cd165b81 100755 --- a/cd/publish_images.sh +++ b/cd/publish_images.sh @@ -101,6 +101,7 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}" pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}" pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}" pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}" +pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}" pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}" pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}" pushContainer gchq/spark-py "${SPARK_VERSION}" From 5e59949427c5d85d83c07a7174a3987663bd2625 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 9 Jan 2024 16:01:55 +0000 Subject: [PATCH 08/12] merge existing demo image with new image --- .gitignore | 1 + docker/gaffer-gremlin/Dockerfile | 1 + docker/gaffer-gremlin/README.md | 38 +++++++++---- docker/gaffer-gremlin/build.sh | 2 +- .../conf/gaffer-gremlin-server.yaml | 8 ++- .../conf}/gaffer/schema/elements.json | 0 .../conf}/gaffer/schema/types.json | 0 .../conf/gaffer/store-accumulo.properties} | 4 +- .../conf/gafferpop/gafferpop.properties | 3 +- .../example/compose.yaml} | 54 +++++-------------- .../gremlin-gaffer-modern-example.ipynb | 29 ++++++---- docker/gaffer-gremlin/pom.xml | 7 --- docker/gremlin-gaffer-demo/.env | 7 --- docker/gremlin-gaffer-demo/Dockerfile | 51 ------------------ docker/gremlin-gaffer-demo/README.md | 39 -------------- .../gafferpop-tinkerpop-modern.properties | 20 ------- .../gremlin-gaffer-demo/conf/graphConfig.json | 3 -- .../conf/gremlin-server-empty-gaffer.yaml | 43 --------------- docker/gremlin-gaffer-demo/files/.gitignore | 2 - 19 files changed, 73 insertions(+), 239 deletions(-) rename docker/{gremlin-gaffer-demo/conf/gafferpop => gaffer-gremlin/conf}/gaffer/schema/elements.json (100%) rename docker/{gremlin-gaffer-demo/conf/gafferpop => gaffer-gremlin/conf}/gaffer/schema/types.json (100%) rename docker/{gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties => gaffer-gremlin/conf/gaffer/store-accumulo.properties} (92%) rename docker/{gremlin-gaffer-demo/docker-compose.yaml => gaffer-gremlin/example/compose.yaml} (76%) rename docker/{gremlin-gaffer-demo => gaffer-gremlin/example}/gremlin-gaffer-modern-example.ipynb (86%) delete mode 100644 docker/gremlin-gaffer-demo/.env delete mode 100644 docker/gremlin-gaffer-demo/Dockerfile delete mode 100644 docker/gremlin-gaffer-demo/README.md delete mode 100644 docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties delete mode 100644 docker/gremlin-gaffer-demo/conf/graphConfig.json delete mode 100644 docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml delete mode 100644 docker/gremlin-gaffer-demo/files/.gitignore diff --git a/.gitignore b/.gitignore index 597bd3b1..fa0d04b4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ **/.DS_Store kubernetes/*/Chart.lock kubernetes/*/charts/ +target/ diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile index 30917f97..de1a5fa6 100644 --- a/docker/gaffer-gremlin/Dockerfile +++ b/docker/gaffer-gremlin/Dockerfile @@ -15,6 +15,7 @@ ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server ARG BASE_IMAGE_TAG=3.7.1 +# Base image provides WORKDIR, USER and ENTRYPOINT FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} # Copy JARS diff --git a/docker/gaffer-gremlin/README.md b/docker/gaffer-gremlin/README.md index 94d8803b..0dfd36a0 100644 --- a/docker/gaffer-gremlin/README.md +++ b/docker/gaffer-gremlin/README.md @@ -2,8 +2,8 @@ ## Build -Run the supplied `build.sh` script with the required environment vars to pull -all dependencies and build the container. +Run the supplied `build.sh` script with the required environment vars (can +source `accumolo2.env` file) to pull all dependencies and build the container. _Note requires Maven and Docker_ @@ -11,12 +11,16 @@ Optionally run each command separately to configure the containers tags etc. ## Configure -The container will use the Gaffer Proxy store to connect to an existing graph -and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)). +The container will use the Gaffer Proxy store by default to connect to an +existing graph and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)). The Gaffer graph the container will connect to can be configured as usual by editing the `store.properties` file, which you can also bind mount over on an -existing image. +existing image. The config file locations are under the predefined workdir +set by the parent gremlin server image the key locations in the image are: + +- `/opt/gremlin-server/conf/gaffer/store.properties` - Override for custom store properties. +- `/opt/gremlin-server/conf/gafferpop/gafferpop.properties` - Override to configure the graph. The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml` this again can be modified as needed or bind mounted over. Please see the @@ -25,14 +29,28 @@ for more information on configuring this image. ## Run -Simply run the container to publish the Gremlin server this can then be -connected to via the address specified in the server's yaml config to use -Gremlin traversal. The server can then be connected via the Gremlin -console e.g. if using python +Simply run the container to publish the Gremlin server making sure to make the +configured port available (port 8182 by default) this can then be connected to +via the address specified in the server's yaml config to use Gremlin traversal. +The server can then be connected via the Gremlin console e.g. if using +`gremlinpython`: ```python from gremlin_python.process.anonymous_traversal_source import traversal g = traversal().withRemote( - DriverRemoteConnection('ws://localhost:8182/gremlin')) + DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')) +``` + +### Demo Deployment + +A demo/example using the tinkerpop 'modern' dataset and accumulo backed Gaffer +is available under the `example` directory. This can be ran using docker compose +to deploy the containers then, the provided jupiter notebook demonstrates how to +connect and some basic queries on the data using `gremlinpython`. + +Run the example with: + +```bash +docker compose up ``` diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh index 34594223..e27a01dd 100755 --- a/docker/gaffer-gremlin/build.sh +++ b/docker/gaffer-gremlin/build.sh @@ -29,7 +29,7 @@ pushd "${SCRIPT_DIR}" || exit 1 mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}" # Build container - docker build -t gaffer-gremlin:"${GAFFER_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" . + docker build -t gchq/gaffer-gremlin:"${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" . # Clean mvn clean diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml index 6dd17d7a..8d9aa562 100644 --- a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml +++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml @@ -29,10 +29,14 @@ scriptEngines: { } } serializers: - - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} + # application/vnd.graphbinary-v1.0 + - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 } + # application/vnd.graphbinary-v1.0-stringd + - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} metrics: { - slf4jReporter: {enabled: true, interval: 180000}} + slf4jReporter: {enabled: true, interval: 180000} +} strictTransactionManagement: false idleConnectionTimeout: 0 keepAliveInterval: 0 diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json b/docker/gaffer-gremlin/conf/gaffer/schema/elements.json similarity index 100% rename from docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/elements.json rename to docker/gaffer-gremlin/conf/gaffer/schema/elements.json diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json b/docker/gaffer-gremlin/conf/gaffer/schema/types.json similarity index 100% rename from docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/schema/types.json rename to docker/gaffer-gremlin/conf/gaffer/schema/types.json diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties similarity index 92% rename from docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties rename to docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties index efb5b466..b82e1a7a 100644 --- a/docker/gremlin-gaffer-demo/conf/gafferpop/gaffer/store.properties +++ b/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties @@ -1,5 +1,5 @@ # -# Copyright 2016-2023 Crown Copyright +# Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,4 +21,4 @@ accumulo.user=root accumulo.password=secret # General store config gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService -gaffer.store.job.tracker.enabled=true +gaffer.store.job.tracker.enabled=true \ No newline at end of file diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties index 156805b0..10e81747 100644 --- a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties +++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties @@ -14,6 +14,7 @@ # limitations under the License. # gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph -gaffer.graphId=graphProxy +gaffer.graphId=graph1 gaffer.storeproperties=conf/gaffer/store.properties +gaffer.schemas=conf/gaffer/schema/ gaffer.userId=user01 diff --git a/docker/gremlin-gaffer-demo/docker-compose.yaml b/docker/gaffer-gremlin/example/compose.yaml similarity index 76% rename from docker/gremlin-gaffer-demo/docker-compose.yaml rename to docker/gaffer-gremlin/example/compose.yaml index 4bb32592..c2a392dd 100644 --- a/docker/gremlin-gaffer-demo/docker-compose.yaml +++ b/docker/gaffer-gremlin/example/compose.yaml @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Crown Copyright +# Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,10 +42,6 @@ services: interval: 30s timeout: 10s retries: 3 - build: - context: ../hdfs/ - args: - HADOOP_VERSION: ${HADOOP_VERSION} command: namenode container_name: hdfs-namenode hostname: hdfs-namenode @@ -54,7 +50,7 @@ services: ports: - 9870:9870 volumes: - - ../hdfs/conf:${HADOOP_CONF_DIR}:ro + - ../../hdfs/conf:${HADOOP_CONF_DIR}:ro - /var/log/hadoop - /data1 - /data2 @@ -70,7 +66,7 @@ services: environment: - HADOOP_CONF_DIR=${HADOOP_CONF_DIR} volumes: - - ../hdfs/conf:${HADOOP_CONF_DIR}:ro + - ../../hdfs/conf:${HADOOP_CONF_DIR}:ro - /var/log/hadoop - /data1 - /data2 @@ -86,12 +82,6 @@ services: timeout: 5s retries: 3 start_period: 10s - build: - context: . - args: - GAFFER_VERSION: ${GAFFER_VERSION} - BASE_IMAGE_NAME: gchq/accumulo - BASE_IMAGE_TAG: ${ACCUMULO_VERSION} command: master container_name: accumulo-master hostname: accumulo-master @@ -103,7 +93,7 @@ services: # accumulo to "auth" with HDFS as the super user so that it can: - HADOOP_USER_NAME=hadoop volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo accumulo-tserver: @@ -127,7 +117,7 @@ services: # accumulo to "auth" with HDFS as the super user so that it can: - HADOOP_USER_NAME=hadoop volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo accumulo-monitor: @@ -148,7 +138,7 @@ services: ports: - 9995:9995 volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo accumulo-gc: @@ -167,37 +157,17 @@ services: # accumulo to "auth" with HDFS as the super user so that it can: - HADOOP_USER_NAME=hadoop volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo - gaffer-rest: - image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + gaffer-gremlin: + image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION} depends_on: accumulo-tserver: condition: service_healthy - build: - context: ../gaffer-rest/ - args: - GAFFER_VERSION: ${GAFFER_VERSION} - ACCUMULO_VERSION: ${ACCUMULO_VERSION} - ports: - - 8080:8080 - volumes: - - ./conf/graphConfig.json:/gaffer/graph/graphConfig.json:ro - - ./conf/gafferpop/gaffer/store.properties:/gaffer/store/store.properties:ro - - ./conf/gafferpop/gaffer/schema:/gaffer/schema:ro - - gremlin-gaffer: - image: gchq/gremlin-gaffer:${GREMLIN_VERSION}-gaffer-${GAFFER_VERSION} - depends_on: - accumulo-tserver: - condition: service_healthy - build: - context: . - args: - GAFFER_VERSION: ${GAFFER_VERSION} - GREMLIN_VERSION: ${GREMLIN_VERSION} ports: - 8182:8182 volumes: - - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro + - ../conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro + - ../conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro + - ../conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro diff --git a/docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb b/docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb similarity index 86% rename from docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb rename to docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb index d0a6bb79..00d5643c 100644 --- a/docker/gremlin-gaffer-demo/gremlin-gaffer-modern-example.ipynb +++ b/docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -30,9 +30,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Create gremlin client to run gremlin scripts directly\n", "client = Client('ws://localhost:8182/gremlin', 'g')\n", @@ -42,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -70,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -79,7 +90,7 @@ "[0.5]" ] }, - "execution_count": 15, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -91,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -100,7 +111,7 @@ "[path[v[1], v[3]], path[v[1], v[4], v[3]]]" ] }, - "execution_count": 16, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml index cbd16a74..61ab0432 100644 --- a/docker/gaffer-gremlin/pom.xml +++ b/docker/gaffer-gremlin/pom.xml @@ -25,7 +25,6 @@ 2.1.0 - 3.7.1 2.36 @@ -42,12 +41,6 @@ jersey-hk2 ${jersey.version} - - - org.apache.tinkerpop - gremlin-core - ${tinkerpop.version} - diff --git a/docker/gremlin-gaffer-demo/.env b/docker/gremlin-gaffer-demo/.env deleted file mode 100644 index b9a28848..00000000 --- a/docker/gremlin-gaffer-demo/.env +++ /dev/null @@ -1,7 +0,0 @@ -ZOOKEEPER_VERSION=3.7.1 -GAFFER_VERSION=2.0.0 -GREMLIN_VERSION=3.6.4 -ACCUMULO_VERSION=2.0.1 -HADOOP_VERSION=3.3.3 -ACCUMULO_CONF_DIR=/etc/accumulo/conf -HADOOP_CONF_DIR=/etc/hadoop/conf diff --git a/docker/gremlin-gaffer-demo/Dockerfile b/docker/gremlin-gaffer-demo/Dockerfile deleted file mode 100644 index ffb80399..00000000 --- a/docker/gremlin-gaffer-demo/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2023 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG BUILDER_IMAGE_NAME=maven -ARG BUILDER_IMAGE_TAG=3.8.4-jdk-8 - -ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server -ARG BASE_IMAGE_TAG=3.6.4 - -FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder - -ARG BASE_IMAGE_TAG -ARG GAFFER_VERSION=2.0.0 -ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2 -ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git - -WORKDIR /jars - -# Allow users to provide their own JAR files -COPY ./files/ . -# Try to download required version from Maven Central, otherwise build from source -RUN allFilesDownloaded="TRUE" && \ - if [ ! -f "./tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ] && [ "${allFilesDownloaded}" = "TRUE" ]; then \ - wget -nv "${GAFFER_DOWNLOAD_URL}/uk/gov/gchq/gaffer/tinkerpop/${GAFFER_VERSION}/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" || allFilesDownloaded="FALSE"; \ - fi && \ - if [ "${allFilesDownloaded}" = "FALSE" ]; then \ - git clone ${GAFFER_GIT_REPO} /tmp/gaffer && \ - cd /tmp/gaffer && \ - git checkout ${GAFFER_VERSION} || git checkout gaffer2-${GAFFER_VERSION} && \ - mvn clean package -Pquick -pl :tinkerpop && \ - if [ ! -f "/jars/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ]; then \ - cp ./library/tinkerpop/target/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar /jars; \ - fi \ - fi - -FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} -COPY --from=builder --chown=root:root /jars/*.jar ext/gafferpop/plugin/ -COPY ./conf/gafferpop/ conf/gafferpop/ -COPY ./conf/gremlin-server-empty-gaffer.yaml conf/gremlin-server-empty-gaffer.yaml -CMD ["conf/gremlin-server-empty-gaffer.yaml"] diff --git a/docker/gremlin-gaffer-demo/README.md b/docker/gremlin-gaffer-demo/README.md deleted file mode 100644 index 018e76d6..00000000 --- a/docker/gremlin-gaffer-demo/README.md +++ /dev/null @@ -1,39 +0,0 @@ -Gremlin Gaffer Plugin -====== -In this folder you can find the required files for building and running a gremlin-server with the Gaffer plugin loaded. - -The Docker image uses TinkerPop's gremlin-server with GafferPop config and plugin jars added in. -When run with docker compose it will provide you a full accumulo ecosystem complete with [hdfs](../hdfs) and the [Gaffer REST API](../gaffer-rest). - -# Running Locally -The easiest way to build and run these services is to use docker compose, by running the following from this directory: -```bash -docker compose up -``` - -## Example Notebook -See `gremlin-gaffer-modern-example.ipynb` for an example using the "TinkerPop Modern" demo graph. - -## Customising the build -Custom Gaffer TinkerPop plugin jars can be added in the files directory. The Gaffer schema, store properties and gafferpop properties can be found in `conf/gafferpop` and are customised in a docker compose build using volumes. The `gremlin-server-empty-gaffer.yaml` cannot be overwritten in a volume, it must be built into the image. - -## Containers that are started: -* Zookeeper -* HDFS - * Datanode - * Namenode -* Accumulo - * Monitor - * GC - * tserver - * Master -* Gaffer REST -* Gremlin Server with GafferPop - -Access the HDFS NameNode web UI at: http://localhost:9870 - -Access the Accumulo Monitor UI at: http://localhost:9995 - -Access the Gaffer REST API at: http://localhost:8080/rest/ - -Access the Gremlin Server with GafferPop at: http://localhost:8182/ diff --git a/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties b/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties deleted file mode 100644 index 2bb1fbfb..00000000 --- a/docker/gremlin-gaffer-demo/conf/gafferpop/gafferpop-tinkerpop-modern.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2016-2023 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph -gaffer.graphId=graph1 -gaffer.storeproperties=conf/gafferpop/gaffer/store.properties -gaffer.schemas=conf/gafferpop/gaffer/schema/ -gaffer.userId=user01 diff --git a/docker/gremlin-gaffer-demo/conf/graphConfig.json b/docker/gremlin-gaffer-demo/conf/graphConfig.json deleted file mode 100644 index 57f97103..00000000 --- a/docker/gremlin-gaffer-demo/conf/graphConfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "graphId": "graph1" -} \ No newline at end of file diff --git a/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml b/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml deleted file mode 100644 index 0fae7ec1..00000000 --- a/docker/gremlin-gaffer-demo/conf/gremlin-server-empty-gaffer.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2023 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -host: localhost -port: 8182 -evaluationTimeout: 30000 -graphs: { - graph: conf/gafferpop/gafferpop-tinkerpop-modern.properties} -scriptEngines: { - gremlin-groovy: { - plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, - uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {}, - org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true}, - org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]}, - org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}} - } -} -serializers: - - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} # application/vnd.graphbinary-v1.0-stringd - -metrics: { - slf4jReporter: {enabled: true, interval: 180000}} -strictTransactionManagement: false -idleConnectionTimeout: 0 -keepAliveInterval: 0 -maxInitialLineLength: 4096 -maxHeaderSize: 8192 -maxChunkSize: 8192 -maxContentLength: 10485760 -maxAccumulationBufferComponents: 1024 -resultIterationBatchSize: 64 -enableAuditLog: true diff --git a/docker/gremlin-gaffer-demo/files/.gitignore b/docker/gremlin-gaffer-demo/files/.gitignore deleted file mode 100644 index fd807ac5..00000000 --- a/docker/gremlin-gaffer-demo/files/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.tar.gz -*.jar From c333d6545a2189d409a5a53eed71ba8dfc5f7899 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Tue, 9 Jan 2024 16:47:30 +0000 Subject: [PATCH 09/12] move build to docker compose and tidy --- docker/gaffer-gremlin/Dockerfile | 2 +- docker/gaffer-gremlin/build.sh | 4 +-- docker/gaffer-gremlin/compose.yaml | 25 +++++++++++++++++++ .../conf/gaffer-gremlin-server.yaml | 2 +- .../conf/gaffer/store.properties | 2 +- .../conf/gafferpop/gafferpop.properties | 2 +- docker/gaffer-gremlin/example/compose.yaml | 6 ++--- .../conf/gaffer/schema/elements.json | 0 .../conf/gaffer/schema/types.json | 0 .../conf/gaffer/store-accumulo.properties | 2 +- .../conf/gafferpop/gafferpop.properties | 20 +++++++++++++++ docker/gaffer-gremlin/pom.xml | 2 +- 12 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 docker/gaffer-gremlin/compose.yaml rename docker/gaffer-gremlin/{ => example}/conf/gaffer/schema/elements.json (100%) rename docker/gaffer-gremlin/{ => example}/conf/gaffer/schema/types.json (100%) rename docker/gaffer-gremlin/{ => example}/conf/gaffer/store-accumulo.properties (96%) create mode 100644 docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile index de1a5fa6..5fa93c14 100644 --- a/docker/gaffer-gremlin/Dockerfile +++ b/docker/gaffer-gremlin/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh index e27a01dd..a35b6872 100755 --- a/docker/gaffer-gremlin/build.sh +++ b/docker/gaffer-gremlin/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ pushd "${SCRIPT_DIR}" || exit 1 mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}" # Build container - docker build -t gchq/gaffer-gremlin:"${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}" --build-arg BASE_IMAGE_TAG="${TINKERPOP_VERSION}" . + docker compose build # Clean mvn clean diff --git a/docker/gaffer-gremlin/compose.yaml b/docker/gaffer-gremlin/compose.yaml new file mode 100644 index 00000000..ad8370f6 --- /dev/null +++ b/docker/gaffer-gremlin/compose.yaml @@ -0,0 +1,25 @@ +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.7" + +services: + gaffer-gremlin: + image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION} + build: + context: . + args: + BASE_IMAGE_TAG: ${TINKERPOP_VERSION} + ports: + - 8182:8182 diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml index 8d9aa562..acebb348 100644 --- a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml +++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/gaffer-gremlin/conf/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store.properties index 2195d783..6dad3cd4 100644 --- a/docker/gaffer-gremlin/conf/gaffer/store.properties +++ b/docker/gaffer-gremlin/conf/gaffer/store.properties @@ -1,5 +1,5 @@ # -# Copyright 2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties index 10e81747..e4e662b0 100644 --- a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties +++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties @@ -1,5 +1,5 @@ # -# Copyright 2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/gaffer-gremlin/example/compose.yaml b/docker/gaffer-gremlin/example/compose.yaml index c2a392dd..51c34e44 100644 --- a/docker/gaffer-gremlin/example/compose.yaml +++ b/docker/gaffer-gremlin/example/compose.yaml @@ -168,6 +168,6 @@ services: ports: - 8182:8182 volumes: - - ../conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro - - ../conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro - - ../conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro + - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro + - ./conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro + - ./conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro diff --git a/docker/gaffer-gremlin/conf/gaffer/schema/elements.json b/docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json similarity index 100% rename from docker/gaffer-gremlin/conf/gaffer/schema/elements.json rename to docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json diff --git a/docker/gaffer-gremlin/conf/gaffer/schema/types.json b/docker/gaffer-gremlin/example/conf/gaffer/schema/types.json similarity index 100% rename from docker/gaffer-gremlin/conf/gaffer/schema/types.json rename to docker/gaffer-gremlin/example/conf/gaffer/schema/types.json diff --git a/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties b/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties similarity index 96% rename from docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties rename to docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties index b82e1a7a..212615d9 100644 --- a/docker/gaffer-gremlin/conf/gaffer/store-accumulo.properties +++ b/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties @@ -1,5 +1,5 @@ # -# Copyright 2024 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties new file mode 100644 index 00000000..e4e662b0 --- /dev/null +++ b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties @@ -0,0 +1,20 @@ +# +# Copyright 2023-2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph +gaffer.graphId=graph1 +gaffer.storeproperties=conf/gaffer/store.properties +gaffer.schemas=conf/gaffer/schema/ +gaffer.userId=user01 diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml index 61ab0432..65f525f3 100644 --- a/docker/gaffer-gremlin/pom.xml +++ b/docker/gaffer-gremlin/pom.xml @@ -1,6 +1,6 @@