diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index fc97fbc4..d59a82b1 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -58,8 +58,8 @@ jobs: run: ./cd/lint_charts.sh - name: Deploy to Kubernetes - run: ./cd/deploy_to_kind.sh - + run: ./cd/deploy_to_kind.sh ./docker/accumulo2.env + - name: Run gaffer-road-traffic Tests run: helm test gaffer || (kubectl get po && kubectl describe po && kubectl logs -l app.kubernetes.io/component=test --tail=-1 && df -h && false) diff --git a/.gitignore b/.gitignore index 597bd3b1..f795c987 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Crown Copyright +# Copyright 2020-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,3 +21,4 @@ **/.DS_Store kubernetes/*/Chart.lock kubernetes/*/charts/ +target/ diff --git a/cd/build_images.sh b/cd/build_images.sh index 9a8b4a08..f69b9ab5 100755 --- a/cd/build_images.sh +++ b/cd/build_images.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2020-2023 Crown Copyright +# Copyright 2020-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,22 +18,23 @@ # Required for variables from sourced env file to automatically be visible to docker compose. set -e -a -root_directory="$( cd $(dirname $(dirname $0)) > /dev/null 2>&1 && pwd )" -cd $root_directory +ROOT_DIR="$(readlink -f "$(dirname "$(dirname "${0}")")")" -if [ ! -z "$1" ]; then - ENV_FILE=$1 -else - echo "Error - Environment file not set"; exit 1; -fi +pushd "${ROOT_DIR}" || exit 1 -# The following command sets: +# The following env file will be sourced to set: # HADOOP_VERSION # GAFFER_VERSION # GAFFERPY_VERSION # ACCUMULO_VERSION # SPARK_VERSION -source "${ENV_FILE}" +# TINKERPOP_VERSION +if [[ -f "${1}" ]]; then + source "${1}" +else + echo "Error - Environment file not set" + exit 1 +fi # Builds all of the Gaffer and Accumulo related images: docker compose --project-directory ./docker/accumulo/ -f ./docker/accumulo/docker-compose.yaml build @@ -41,8 +42,12 @@ docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./doc # Builds all of the notebook related images: docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build +# Builds the Gaffer Gremlin server +./docker/gaffer-gremlin/build.sh # Set $JHUB_OPTIONS_SERVER_VERSION source ./docker/gaffer-jhub-options-server/get-version.sh # Builds the jhub options server: docker compose --project-directory ./docker/gaffer-jhub-options-server/ -f ./docker/gaffer-jhub-options-server/docker-compose.yaml build + +popd || exit 1 diff --git a/cd/deploy_to_kind.sh b/cd/deploy_to_kind.sh index 68c7cfa8..12846083 100755 --- a/cd/deploy_to_kind.sh +++ b/cd/deploy_to_kind.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2020-2023 Crown Copyright +# Copyright 2020-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,13 @@ kind create cluster --quiet --config ./cd/kind.yaml --image kindest/node:v1.24.4 # HADOOP_VERSION # GAFFER_VERSION # SPARK_VERSION -source ./docker/gaffer-pyspark-notebook/.env +# KUBECTL_VERSION +if [[ -f "${1}" ]]; then + source "${1}" +else + echo "Error - Environment file not set" + exit 1 +fi # JHUB_OPTIONS_SERVER_VERSION source ./docker/gaffer-jhub-options-server/get-version.sh diff --git a/cd/publish_images.sh b/cd/publish_images.sh index 86c26061..90adcff5 100755 --- a/cd/publish_images.sh +++ b/cd/publish_images.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2020-2023 Crown Copyright +# Copyright 2020-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,6 +101,7 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}" pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}" pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}" pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}" +pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}" pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}" pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}" pushContainer gchq/spark-py "${SPARK_VERSION}" diff --git a/cd/update_versions.sh b/cd/update_versions.sh index 2479bd1c..d96f1c81 100755 --- a/cd/update_versions.sh +++ b/cd/update_versions.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2020-2023 Crown Copyright +# Copyright 2020-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,9 +23,9 @@ fi # GAFFER_VERSION # GAFFERPY_VERSION # SPARK_VERSION -source ./docker/gaffer-pyspark-notebook/.env +source ./docker/accumulo2.env # JHUB_OPTIONS_SERVER_VERSION -source docker/gaffer-jhub-options-server/get-version.sh +source ./docker/gaffer-jhub-options-server/get-version.sh # hdfs [ ! -z "${APP_VERSION}" ] && yq eval ".version = \"${APP_VERSION}\"" -i ./kubernetes/hdfs/Chart.yaml diff --git a/docker/accumulo2.env b/docker/accumulo2.env index e415ae8f..585fa420 100644 --- a/docker/accumulo2.env +++ b/docker/accumulo2.env @@ -7,3 +7,5 @@ GAFFER_VERSION=2.1.0 GAFFER_TESTER_VERSION=2.1.0 GAFFERPY_VERSION=2.1.0 SPARK_VERSION=3.1.2 +TINKERPOP_VERSION=3.7.1 +KUBECTL_VERSION=1.23.0 diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile new file mode 100644 index 00000000..5fa93c14 --- /dev/null +++ b/docker/gaffer-gremlin/Dockerfile @@ -0,0 +1,28 @@ +# Copyright 2023-2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server +ARG BASE_IMAGE_TAG=3.7.1 + +# Base image provides WORKDIR, USER and ENTRYPOINT +FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} + +# Copy JARS +COPY ./target/dependency/*.jar ext/gafferpop/plugin/ + +# Copy configs +COPY ./conf/ conf/ + +# Set server to run +CMD ["conf/gaffer-gremlin-server.yaml"] diff --git a/docker/gaffer-gremlin/README.md b/docker/gaffer-gremlin/README.md new file mode 100644 index 00000000..0dfd36a0 --- /dev/null +++ b/docker/gaffer-gremlin/README.md @@ -0,0 +1,56 @@ +# gaffer-gremlin + +## Build + +Run the supplied `build.sh` script with the required environment vars (can +source `accumolo2.env` file) to pull all dependencies and build the container. + +_Note requires Maven and Docker_ + +Optionally run each command separately to configure the containers tags etc. + +## Configure + +The container will use the Gaffer Proxy store by default to connect to an +existing graph and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)). + +The Gaffer graph the container will connect to can be configured as usual by +editing the `store.properties` file, which you can also bind mount over on an +existing image. The config file locations are under the predefined workdir +set by the parent gremlin server image the key locations in the image are: + +- `/opt/gremlin-server/conf/gaffer/store.properties` - Override for custom store properties. +- `/opt/gremlin-server/conf/gafferpop/gafferpop.properties` - Override to configure the graph. + +The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml` +this again can be modified as needed or bind mounted over. Please see the +[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/) +for more information on configuring this image. + +## Run + +Simply run the container to publish the Gremlin server making sure to make the +configured port available (port 8182 by default) this can then be connected to +via the address specified in the server's yaml config to use Gremlin traversal. +The server can then be connected via the Gremlin console e.g. if using +`gremlinpython`: + +```python +from gremlin_python.process.anonymous_traversal_source import traversal + +g = traversal().withRemote( + DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')) +``` + +### Demo Deployment + +A demo/example using the tinkerpop 'modern' dataset and accumulo backed Gaffer +is available under the `example` directory. This can be ran using docker compose +to deploy the containers then, the provided jupiter notebook demonstrates how to +connect and some basic queries on the data using `gremlinpython`. + +Run the example with: + +```bash +docker compose up +``` diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh new file mode 100755 index 00000000..a35b6872 --- /dev/null +++ b/docker/gaffer-gremlin/build.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2023-2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")" + +# Check if already have env var for version +[[ -z "${GAFFER_VERSION}" ]] && \ + echo "Missing GAFFER_VERSION env var" && \ + exit 1 +[[ -z "${TINKERPOP_VERSION}" ]] && \ + echo "Missing TINKERPOP_VERSION env var" && \ + exit 1 + +# Build from relevant directory +pushd "${SCRIPT_DIR}" || exit 1 + # Download JARs + mvn clean dependency:copy-dependencies --define gaffer.version="${GAFFER_VERSION}" + + # Build container + docker compose build + + # Clean + mvn clean +popd || exit 1 + +echo "Build Successful" diff --git a/docker/gaffer-gremlin/compose.yaml b/docker/gaffer-gremlin/compose.yaml new file mode 100644 index 00000000..ad8370f6 --- /dev/null +++ b/docker/gaffer-gremlin/compose.yaml @@ -0,0 +1,25 @@ +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.7" + +services: + gaffer-gremlin: + image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION} + build: + context: . + args: + BASE_IMAGE_TAG: ${TINKERPOP_VERSION} + ports: + - 8182:8182 diff --git a/docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml similarity index 74% rename from docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml rename to docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml index 0fae7ec1..acebb348 100644 --- a/docker/gremlin-gaffer/conf/gremlin-server-empty-gaffer.yaml +++ b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,21 +16,27 @@ host: localhost port: 8182 evaluationTimeout: 30000 graphs: { - graph: conf/gafferpop/gafferpop-tinkerpop-modern.properties} + graph: conf/gafferpop/gafferpop.properties +} scriptEngines: { gremlin-groovy: { plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {}, org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: {enableThreadInterrupt: true}, org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]}, - org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]}} + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/empty-sample.groovy]} + } } } serializers: - - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} # application/vnd.graphbinary-v1.0-stringd + # application/vnd.graphbinary-v1.0 + - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 } + # application/vnd.graphbinary-v1.0-stringd + - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} metrics: { - slf4jReporter: {enabled: true, interval: 180000}} + slf4jReporter: {enabled: true, interval: 180000} +} strictTransactionManagement: false idleConnectionTimeout: 0 keepAliveInterval: 0 diff --git a/docker/gaffer-gremlin/conf/gaffer/store.properties b/docker/gaffer-gremlin/conf/gaffer/store.properties new file mode 100644 index 00000000..6dad3cd4 --- /dev/null +++ b/docker/gaffer-gremlin/conf/gaffer/store.properties @@ -0,0 +1,20 @@ +# +# Copyright 2023-2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore +# These can be configured to an existing graph deployment +gaffer.host=localhost +gaffer.port=8080 +gaffer.context-root=/rest/latest diff --git a/docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties similarity index 81% rename from docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties rename to docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties index 2bb1fbfb..e4e662b0 100644 --- a/docker/gremlin-gaffer/conf/gafferpop/gafferpop-tinkerpop-modern.properties +++ b/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties @@ -1,5 +1,5 @@ # -# Copyright 2016-2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,6 @@ # gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph gaffer.graphId=graph1 -gaffer.storeproperties=conf/gafferpop/gaffer/store.properties -gaffer.schemas=conf/gafferpop/gaffer/schema/ +gaffer.storeproperties=conf/gaffer/store.properties +gaffer.schemas=conf/gaffer/schema/ gaffer.userId=user01 diff --git a/docker/gremlin-gaffer/docker-compose.yaml b/docker/gaffer-gremlin/example/compose.yaml similarity index 77% rename from docker/gremlin-gaffer/docker-compose.yaml rename to docker/gaffer-gremlin/example/compose.yaml index 4bb32592..51c34e44 100644 --- a/docker/gremlin-gaffer/docker-compose.yaml +++ b/docker/gaffer-gremlin/example/compose.yaml @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Crown Copyright +# Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,10 +42,6 @@ services: interval: 30s timeout: 10s retries: 3 - build: - context: ../hdfs/ - args: - HADOOP_VERSION: ${HADOOP_VERSION} command: namenode container_name: hdfs-namenode hostname: hdfs-namenode @@ -54,7 +50,7 @@ services: ports: - 9870:9870 volumes: - - ../hdfs/conf:${HADOOP_CONF_DIR}:ro + - ../../hdfs/conf:${HADOOP_CONF_DIR}:ro - /var/log/hadoop - /data1 - /data2 @@ -70,7 +66,7 @@ services: environment: - HADOOP_CONF_DIR=${HADOOP_CONF_DIR} volumes: - - ../hdfs/conf:${HADOOP_CONF_DIR}:ro + - ../../hdfs/conf:${HADOOP_CONF_DIR}:ro - /var/log/hadoop - /data1 - /data2 @@ -86,12 +82,6 @@ services: timeout: 5s retries: 3 start_period: 10s - build: - context: . - args: - GAFFER_VERSION: ${GAFFER_VERSION} - BASE_IMAGE_NAME: gchq/accumulo - BASE_IMAGE_TAG: ${ACCUMULO_VERSION} command: master container_name: accumulo-master hostname: accumulo-master @@ -103,7 +93,7 @@ services: # accumulo to "auth" with HDFS as the super user so that it can: - HADOOP_USER_NAME=hadoop volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo accumulo-tserver: @@ -127,7 +117,7 @@ services: # accumulo to "auth" with HDFS as the super user so that it can: - HADOOP_USER_NAME=hadoop volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo accumulo-monitor: @@ -148,7 +138,7 @@ services: ports: - 9995:9995 volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo accumulo-gc: @@ -167,37 +157,17 @@ services: # accumulo to "auth" with HDFS as the super user so that it can: - HADOOP_USER_NAME=hadoop volumes: - - ../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro + - ../../accumulo/conf-${ACCUMULO_VERSION}:${ACCUMULO_CONF_DIR}:ro - /var/log/accumulo - gaffer-rest: - image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + gaffer-gremlin: + image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION} depends_on: accumulo-tserver: condition: service_healthy - build: - context: ../gaffer-rest/ - args: - GAFFER_VERSION: ${GAFFER_VERSION} - ACCUMULO_VERSION: ${ACCUMULO_VERSION} - ports: - - 8080:8080 - volumes: - - ./conf/graphConfig.json:/gaffer/graph/graphConfig.json:ro - - ./conf/gafferpop/gaffer/store.properties:/gaffer/store/store.properties:ro - - ./conf/gafferpop/gaffer/schema:/gaffer/schema:ro - - gremlin-gaffer: - image: gchq/gremlin-gaffer:${GREMLIN_VERSION}-gaffer-${GAFFER_VERSION} - depends_on: - accumulo-tserver: - condition: service_healthy - build: - context: . - args: - GAFFER_VERSION: ${GAFFER_VERSION} - GREMLIN_VERSION: ${GREMLIN_VERSION} ports: - 8182:8182 volumes: - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro + - ./conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro + - ./conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json b/docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json similarity index 100% rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/elements.json rename to docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json b/docker/gaffer-gremlin/example/conf/gaffer/schema/types.json similarity index 100% rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/schema/types.json rename to docker/gaffer-gremlin/example/conf/gaffer/schema/types.json diff --git a/docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties b/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties similarity index 92% rename from docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties rename to docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties index efb5b466..212615d9 100644 --- a/docker/gremlin-gaffer/conf/gafferpop/gaffer/store.properties +++ b/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties @@ -1,5 +1,5 @@ # -# Copyright 2016-2023 Crown Copyright +# Copyright 2023-2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,4 +21,4 @@ accumulo.user=root accumulo.password=secret # General store config gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService -gaffer.store.job.tracker.enabled=true +gaffer.store.job.tracker.enabled=true \ No newline at end of file diff --git a/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties new file mode 100644 index 00000000..e4e662b0 --- /dev/null +++ b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties @@ -0,0 +1,20 @@ +# +# Copyright 2023-2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph +gaffer.graphId=graph1 +gaffer.storeproperties=conf/gaffer/store.properties +gaffer.schemas=conf/gaffer/schema/ +gaffer.userId=user01 diff --git a/docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb b/docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb similarity index 86% rename from docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb rename to docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb index d0a6bb79..00d5643c 100644 --- a/docker/gremlin-gaffer/gremlin-gaffer-modern-example.ipynb +++ b/docker/gaffer-gremlin/example/gremlin-gaffer-modern-example.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -30,9 +30,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Create gremlin client to run gremlin scripts directly\n", "client = Client('ws://localhost:8182/gremlin', 'g')\n", @@ -42,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -70,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -79,7 +90,7 @@ "[0.5]" ] }, - "execution_count": 15, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -91,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -100,7 +111,7 @@ "[path[v[1], v[3]], path[v[1], v[4], v[3]]]" ] }, - "execution_count": 16, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml new file mode 100644 index 00000000..65f525f3 --- /dev/null +++ b/docker/gaffer-gremlin/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + uk.gov.gchq.gaffer.docker + gaffer-gremlin + 0.0.1-SNAPSHOT + pom + + + + 2.1.0 + 2.36 + + + + + + uk.gov.gchq.gaffer + tinkerpop + ${gaffer.version} + + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} + + + + diff --git a/docker/gremlin-gaffer/.env b/docker/gremlin-gaffer/.env deleted file mode 100644 index e273dcd5..00000000 --- a/docker/gremlin-gaffer/.env +++ /dev/null @@ -1,7 +0,0 @@ -ZOOKEEPER_VERSION=3.7.1 -GAFFER_VERSION=2.1.0 -GREMLIN_VERSION=3.6.4 -ACCUMULO_VERSION=2.0.1 -HADOOP_VERSION=3.3.3 -ACCUMULO_CONF_DIR=/etc/accumulo/conf -HADOOP_CONF_DIR=/etc/hadoop/conf diff --git a/docker/gremlin-gaffer/Dockerfile b/docker/gremlin-gaffer/Dockerfile deleted file mode 100644 index d654c610..00000000 --- a/docker/gremlin-gaffer/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2023-2024 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG BUILDER_IMAGE_NAME=maven -ARG BUILDER_IMAGE_TAG=3.8.4-jdk-8 - -ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server -ARG BASE_IMAGE_TAG=3.6.4 - -FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder - -ARG BASE_IMAGE_TAG -ARG GAFFER_VERSION=2.1.0 -ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2 -ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git - -WORKDIR /jars - -# Allow users to provide their own JAR files -COPY ./files/ . -# Try to download required version from Maven Central, otherwise build from source -RUN allFilesDownloaded="TRUE" && \ - if [ ! -f "./tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ] && [ "${allFilesDownloaded}" = "TRUE" ]; then \ - wget -nv "${GAFFER_DOWNLOAD_URL}/uk/gov/gchq/gaffer/tinkerpop/${GAFFER_VERSION}/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" || allFilesDownloaded="FALSE"; \ - fi && \ - if [ "${allFilesDownloaded}" = "FALSE" ]; then \ - git clone ${GAFFER_GIT_REPO} /tmp/gaffer && \ - cd /tmp/gaffer && \ - git checkout ${GAFFER_VERSION} || git checkout gaffer2-${GAFFER_VERSION} && \ - mvn clean package -Pquick -pl :tinkerpop && \ - if [ ! -f "/jars/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar" ]; then \ - cp ./library/tinkerpop/target/tinkerpop-${GAFFER_VERSION}-jar-with-dependencies.jar /jars; \ - fi \ - fi - -FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} -COPY --from=builder --chown=root:root /jars/*.jar ext/gafferpop/plugin/ -COPY ./conf/gafferpop/ conf/gafferpop/ -COPY ./conf/gremlin-server-empty-gaffer.yaml conf/gremlin-server-empty-gaffer.yaml -CMD ["conf/gremlin-server-empty-gaffer.yaml"] diff --git a/docker/gremlin-gaffer/README.md b/docker/gremlin-gaffer/README.md deleted file mode 100644 index 018e76d6..00000000 --- a/docker/gremlin-gaffer/README.md +++ /dev/null @@ -1,39 +0,0 @@ -Gremlin Gaffer Plugin -====== -In this folder you can find the required files for building and running a gremlin-server with the Gaffer plugin loaded. - -The Docker image uses TinkerPop's gremlin-server with GafferPop config and plugin jars added in. -When run with docker compose it will provide you a full accumulo ecosystem complete with [hdfs](../hdfs) and the [Gaffer REST API](../gaffer-rest). - -# Running Locally -The easiest way to build and run these services is to use docker compose, by running the following from this directory: -```bash -docker compose up -``` - -## Example Notebook -See `gremlin-gaffer-modern-example.ipynb` for an example using the "TinkerPop Modern" demo graph. - -## Customising the build -Custom Gaffer TinkerPop plugin jars can be added in the files directory. The Gaffer schema, store properties and gafferpop properties can be found in `conf/gafferpop` and are customised in a docker compose build using volumes. The `gremlin-server-empty-gaffer.yaml` cannot be overwritten in a volume, it must be built into the image. - -## Containers that are started: -* Zookeeper -* HDFS - * Datanode - * Namenode -* Accumulo - * Monitor - * GC - * tserver - * Master -* Gaffer REST -* Gremlin Server with GafferPop - -Access the HDFS NameNode web UI at: http://localhost:9870 - -Access the Accumulo Monitor UI at: http://localhost:9995 - -Access the Gaffer REST API at: http://localhost:8080/rest/ - -Access the Gremlin Server with GafferPop at: http://localhost:8182/ diff --git a/docker/gremlin-gaffer/conf/graphConfig.json b/docker/gremlin-gaffer/conf/graphConfig.json deleted file mode 100644 index 57f97103..00000000 --- a/docker/gremlin-gaffer/conf/graphConfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "graphId": "graph1" -} \ No newline at end of file diff --git a/docker/gremlin-gaffer/files/.gitignore b/docker/gremlin-gaffer/files/.gitignore deleted file mode 100644 index fd807ac5..00000000 --- a/docker/gremlin-gaffer/files/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.tar.gz -*.jar