From 2c34291661c3803a8f09840c25f05311da4b6de4 Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:43:53 +0000 Subject: [PATCH] update proxy example --- .../gremlinpython-modern-example.ipynb | 2 +- .../conf/gaffer/gafferpop.properties | 17 +++ .../conf/gaffer/schema/elements.json | 2 +- .../modern-example/create-deployment.sh | 4 +- .../add-tinkerpop-modern.json} | 0 .../proxy-example/README.md | 23 ++-- .../proxy-example/compose.yaml | 14 +- .../proxy-example/conf/map/store.properties | 19 +++ .../conf/proxy/gafferpop.properties | 17 +++ .../proxy}/operationsDeclarations.json | 0 .../{config => conf/proxy}/store.properties | 4 +- .../proxy-example/conf/schema/elements.json | 49 +++++++ .../proxy-example/conf/schema/types.json | 43 +++++++ .../proxy-example/create-deployment.sh | 59 +++++++++ .../operations/add-tinkerpop-modern.json | 121 ++++++++++++++++++ 15 files changed, 353 insertions(+), 21 deletions(-) create mode 100644 docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties rename docker/example-deployments/modern-example/{conf/gaffer/data/tinkerpop-modern.json => operations/add-tinkerpop-modern.json} (100%) create mode 100644 docker/example-deployments/proxy-example/conf/map/store.properties create mode 100644 docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties rename docker/example-deployments/proxy-example/{config => conf/proxy}/operationsDeclarations.json (100%) rename docker/example-deployments/proxy-example/{config => conf/proxy}/store.properties (95%) create mode 100644 docker/example-deployments/proxy-example/conf/schema/elements.json create mode 100644 docker/example-deployments/proxy-example/conf/schema/types.json create mode 100755 docker/example-deployments/proxy-example/create-deployment.sh create mode 100644 docker/example-deployments/proxy-example/operations/add-tinkerpop-modern.json diff --git a/docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb b/docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb index 149429cd..29ba70a0 100644 --- a/docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb +++ b/docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb @@ -104,7 +104,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Run a seeded cypher query via Gremlin" + "Run a seeded cypher query via Gremlin. Submitted as a script rather than via gremlin-python bindings" ] }, { diff --git a/docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties b/docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties new file mode 100644 index 00000000..51c61b73 --- /dev/null +++ b/docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties @@ -0,0 +1,17 @@ +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Default operation config +gaffer.elements.getalllimit=5000 +gaffer.elements.hasstepfilterstage=PRE_AGGREGATION diff --git a/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json b/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json index 97f6285d..9764ec23 100644 --- a/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json +++ b/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json @@ -1,6 +1,6 @@ { "entities": { - "vertex": { + "id": { "vertex": "string.id" }, "person": { diff --git a/docker/example-deployments/modern-example/create-deployment.sh b/docker/example-deployments/modern-example/create-deployment.sh index bf531a09..df261c58 100755 --- a/docker/example-deployments/modern-example/create-deployment.sh +++ b/docker/example-deployments/modern-example/create-deployment.sh @@ -14,7 +14,7 @@ # limitations under the License. CURRENT_DIR="$(dirname "$(readlink -f "${0}")")" -TINKERPOP_MODERN_JSON="${CURRENT_DIR}/conf/gaffer/data/tinkerpop-modern.json" +TINKERPOP_MODERN_JSON="${CURRENT_DIR}/operations/add-tinkerpop-modern.json" GAFFER_REST_URL="http://localhost:8080" # Make sure we have an evironment file for the image versions @@ -56,4 +56,4 @@ then fi echo "-----------------------------------" -echo "Deployment started!" +echo "Deployment started! REST API: ${GAFFER_REST_URL}" diff --git a/docker/example-deployments/modern-example/conf/gaffer/data/tinkerpop-modern.json b/docker/example-deployments/modern-example/operations/add-tinkerpop-modern.json similarity index 100% rename from docker/example-deployments/modern-example/conf/gaffer/data/tinkerpop-modern.json rename to docker/example-deployments/modern-example/operations/add-tinkerpop-modern.json diff --git a/docker/example-deployments/proxy-example/README.md b/docker/example-deployments/proxy-example/README.md index 627fbcf4..e6401b5d 100644 --- a/docker/example-deployments/proxy-example/README.md +++ b/docker/example-deployments/proxy-example/README.md @@ -2,20 +2,21 @@ ## Running -To run it from the `proxy-example` directory you'll need to specify the -environment file path: +This is an example deployment of Gaffer using a Proxy store. This uses two REST +APIs on a Map store with data in and another as a proxy pointing to the other. -```bash -docker compose --env-file ../.env up -``` +The example uses the +[Tinkerpop Modern Graph](https://tinkerpop.apache.org/docs/current/tutorials/the-gremlin-console/#toy-graphs) +as its data set to demonstrate basic querying of data. + +## Running the Example -Or to run it from this directory, the compose file path and environment file -path: +To run the example please use the provided start script with an environment file +to specify the image versions e.g. `accumulo2.env`: ```bash -docker compose -f proxy-example/compose.yaml --env-file .env up +./create-deployment.sh ../../accumulo2.env ``` -With this deployment both a standard Gaffer REST API and a Gaffer REST API using -the Proxy Store will be started. Only the port for the Proxy REST endpoint is -exposed, this will forward operations to the standard Gaffer REST Endpoint. +With the demo deployment the Proxy backed Gaffer REST API will be started on port 8080 +and the Map store backed instance on 8081. diff --git a/docker/example-deployments/proxy-example/compose.yaml b/docker/example-deployments/proxy-example/compose.yaml index 19926e0d..811b3f84 100644 --- a/docker/example-deployments/proxy-example/compose.yaml +++ b/docker/example-deployments/proxy-example/compose.yaml @@ -19,7 +19,12 @@ services: healthcheck: test: sleep 15 timeout: 20s + ports: + - 8081:8080 restart: always + volumes: + - ./conf/map/store.properties:/gaffer/store/store.properties:ro + - ./conf/schema:/gaffer/schema:ro gaffer-rest-proxy: image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} @@ -27,8 +32,9 @@ services: gaffer-rest: condition: service_healthy restart: always - ports: - - 8080:8080 + network_mode: "host" volumes: - - ./config/store.properties:/gaffer/store/store.properties:ro - - ./config/operationsDeclarations.json:/gaffer/store/operationsDeclarations.json:ro + - ./conf/proxy/store.properties:/gaffer/store/store.properties:ro + - ./conf/proxy/operationsDeclarations.json:/gaffer/store/operationsDeclarations.json:ro + - ./conf/proxy/gafferpop.properties:/gaffer/gafferpop.properties:ro + - ./conf/schema:/gaffer/schema:ro diff --git a/docker/example-deployments/proxy-example/conf/map/store.properties b/docker/example-deployments/proxy-example/conf/map/store.properties new file mode 100644 index 00000000..4b0566f3 --- /dev/null +++ b/docker/example-deployments/proxy-example/conf/map/store.properties @@ -0,0 +1,19 @@ +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +gaffer.store.class=uk.gov.gchq.gaffer.mapstore.MapStore +gaffer.store.properties.class=uk.gov.gchq.gaffer.mapstore.MapStoreProperties + +# General store config +gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService +gaffer.store.job.tracker.enabled=true diff --git a/docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties b/docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties new file mode 100644 index 00000000..51c61b73 --- /dev/null +++ b/docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties @@ -0,0 +1,17 @@ +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Default operation config +gaffer.elements.getalllimit=5000 +gaffer.elements.hasstepfilterstage=PRE_AGGREGATION diff --git a/docker/example-deployments/proxy-example/config/operationsDeclarations.json b/docker/example-deployments/proxy-example/conf/proxy/operationsDeclarations.json similarity index 100% rename from docker/example-deployments/proxy-example/config/operationsDeclarations.json rename to docker/example-deployments/proxy-example/conf/proxy/operationsDeclarations.json diff --git a/docker/example-deployments/proxy-example/config/store.properties b/docker/example-deployments/proxy-example/conf/proxy/store.properties similarity index 95% rename from docker/example-deployments/proxy-example/config/store.properties rename to docker/example-deployments/proxy-example/conf/proxy/store.properties index 49bd9c5d..27271e01 100644 --- a/docker/example-deployments/proxy-example/config/store.properties +++ b/docker/example-deployments/proxy-example/conf/proxy/store.properties @@ -16,6 +16,6 @@ gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore gaffer.store.operation.declarations=/gaffer/store/operationsDeclarations.json # Proxy config -gaffer.host=gaffer-rest -gaffer.port=8080 +gaffer.host=localhost +gaffer.port=8081 gaffer.context-root=/rest diff --git a/docker/example-deployments/proxy-example/conf/schema/elements.json b/docker/example-deployments/proxy-example/conf/schema/elements.json new file mode 100644 index 00000000..97f6285d --- /dev/null +++ b/docker/example-deployments/proxy-example/conf/schema/elements.json @@ -0,0 +1,49 @@ +{ + "entities": { + "vertex": { + "vertex": "string.id" + }, + "person": { + "vertex": "string.id", + "properties": { + "name": "property.string", + "age": "property.integer" + } + }, + "software": { + "vertex": "string.id", + "properties": { + "name": "property.string", + "lang": "property.string" + } + } + }, + "edges": { + "knows": { + "source": "string.id", + "destination": "string.id", + "directed": "true", + "properties": { + "weight": "property.double" + } + }, + "created": { + "source": "string.id", + "destination": "string.id", + "directed": "true", + "properties": { + "weight": "property.double" + } + }, + "dependsOn": { + "source": "string.id", + "destination": "string.id", + "directed": "true" + }, + "encapsulates": { + "source": "string.id", + "destination": "string.id", + "directed": "true" + } + } +} diff --git a/docker/example-deployments/proxy-example/conf/schema/types.json b/docker/example-deployments/proxy-example/conf/schema/types.json new file mode 100644 index 00000000..2f632371 --- /dev/null +++ b/docker/example-deployments/proxy-example/conf/schema/types.json @@ -0,0 +1,43 @@ +{ + "types": { + "int.id": { + "class": "java.lang.Integer" + }, + "long.id": { + "class": "java.lang.Long" + }, + "string.id": { + "class": "java.lang.String" + }, + "uuid.id": { + "class": "java.util.UUID" + }, + "true": { + "class": "java.lang.Boolean" + }, + "property.string": { + "class": "java.lang.String", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + }, + "property.integer": { + "class": "java.lang.Integer", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + }, + "property.double": { + "class": "java.lang.Double", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + }, + "property.float": { + "class": "java.lang.Float", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + } + } +} diff --git a/docker/example-deployments/proxy-example/create-deployment.sh b/docker/example-deployments/proxy-example/create-deployment.sh new file mode 100755 index 00000000..f0eff1d7 --- /dev/null +++ b/docker/example-deployments/proxy-example/create-deployment.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CURRENT_DIR="$(dirname "$(readlink -f "${0}")")" +TINKERPOP_MODERN_JSON="${CURRENT_DIR}/operations/add-tinkerpop-modern.json" +GAFFER_REST_URL="http://localhost:8081" + +# Make sure we have an evironment file for the image versions +if [ ! -f "${1}" ]; then + echo "Error - Environment file not set" + echo "Example: ${0} .env" + exit 1 +fi + +# Remove any existing containers +docker compose --env-file "${1}" -f "${CURRENT_DIR}/compose.yaml" down + +# Start new deployment +echo "-----------------------------------" +echo "Starting new deployment..." +docker compose --env-file "${1}" -f "${CURRENT_DIR}/compose.yaml" up --detach + +# Check can connect to instance +echo "-----------------------------------" +echo "Attempting to connect to Gaffer instance..." +if ! docker run --rm --network=host curlimages/curl:latest \ + curl --retry 20 \ + --retry-delay 5 \ + --retry-all-errors "${GAFFER_REST_URL}"/rest/graph/status +then + echo "Failed to connect to Gaffer instance" + exit 1 +fi + +# Load dataset +echo "-----------------------------------" +echo "Loading data into Gaffer instance..." +if ! docker run --rm --network=host curlimages/curl:latest \ + curl -H "Content-Type: application/json" \ + --data "$(cat "${TINKERPOP_MODERN_JSON}")" "${GAFFER_REST_URL}"/rest/graph/operations/execute +then + echo "Failed to load tinkerpop modern JSON: ${TINKERPOP_MODERN_JSON}" + exit 1 +fi + +echo "-----------------------------------" +echo "Deployment started!" diff --git a/docker/example-deployments/proxy-example/operations/add-tinkerpop-modern.json b/docker/example-deployments/proxy-example/operations/add-tinkerpop-modern.json new file mode 100644 index 00000000..b541d267 --- /dev/null +++ b/docker/example-deployments/proxy-example/operations/add-tinkerpop-modern.json @@ -0,0 +1,121 @@ +{ + "class": "AddElements", + "input": [ + { + "class": "Entity", + "group": "person", + "vertex": "1", + "properties": { + "name": "marko", + "age": 29 + } + }, + { + "class": "Entity", + "group": "person", + "vertex": "2", + "properties": { + "name": "vadas", + "age": 27 + } + }, + { + "class": "Entity", + "group": "person", + "vertex": "4", + "properties": { + "name": "josh", + "age": 32 + } + }, + { + "class": "Entity", + "group": "person", + "vertex": "6", + "properties": { + "name": "peter", + "age": 35 + } + }, + { + "class": "Entity", + "group": "software", + "vertex": "3", + "properties": { + "name": "lop", + "lang": "java" + } + }, + { + "class": "Entity", + "group": "software", + "vertex": "5", + "properties": { + "name": "ripple", + "lang": "java" + } + }, + { + "class": "Edge", + "group": "knows", + "source": "1", + "destination": "2", + "directed": true, + "properties": { + "weight": 0.5 + } + }, + { + "class": "Edge", + "group": "created", + "source": "1", + "destination": "3", + "directed": true, + "properties": { + "weight": 0.4 + } + }, + { + "class": "Edge", + "group": "knows", + "source": "1", + "destination": "4", + "directed": true, + "properties": { + "weight": 1.0 + } + }, + { + "class": "Edge", + "group": "created", + "source": "4", + "destination": "3", + "directed": true, + "properties": { + "weight": 0.4 + } + }, + { + "class": "Edge", + "group": "created", + "source": "4", + "destination": "5", + "directed": true, + "properties": { + "weight": 1.0 + } + }, + { + "class": "Edge", + "group": "created", + "source": "6", + "destination": "3", + "directed": true, + "properties": { + "weight": 0.2 + } + } + ], + "skipInvalidElements": false, + "validate": true +} \ No newline at end of file