From 87f5d48b6b60f0bab762d3f96e24ae5f0b05f50d Mon Sep 17 00:00:00 2001 From: tb06904 <141412860+tb06904@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:17:05 +0000 Subject: [PATCH] Gh-381: Migrate and update examples (#382) * Move examples to common location * clear notebooks * update proxy example * update main readme * remove gaffer-gremlin build * Apply suggestions from code review Co-authored-by: cn337131 <141730190+cn337131@users.noreply.github.com> Co-authored-by: p29876 <165825455+p29876@users.noreply.github.com> * address comments --------- Co-authored-by: cn337131 <141730190+cn337131@users.noreply.github.com> Co-authored-by: p29876 <165825455+p29876@users.noreply.github.com> --- README.md | 42 +++--- cd/build_core_images.sh | 1 - cd/build_images.sh | 3 - cd/publish_images.sh | 1 - cd/update_versions.sh | 1 - docker/accumulo1.env | 1 - docker/accumulo2.env | 1 - docker/example-deployments/README.md | 25 ++++ .../graphnotebook-modern-example.ipynb | 2 +- .../gremlinpython-modern-example.ipynb | 22 +++- .../federated-example/README.md | 0 .../federated-example/compose.yaml | 9 +- .../conf/gaffer/gafferpop.properties} | 17 +-- .../conf/gaffer}/store/defaultGraphIds.json | 0 .../conf/gaffer}/store/mergeFunctions.json | 0 .../conf/gaffer}/store/store.properties | 0 .../federated-example/create-deployment.sh | 0 .../operations/add-modern-created-graph.json | 0 .../operations/add-modern-knows-graph.json | 0 .../modern-example/README.md | 26 ++++ .../modern-example}/compose.yaml | 13 +- .../conf/gaffer}/gafferpop.properties | 11 +- .../conf/gaffer/schema/elements.json | 49 +++++++ .../conf/gaffer/schema/types.json | 0 .../conf/gaffer/store-accumulo.properties | 2 +- .../modern-example/create-deployment.sh | 59 +++++++++ .../operations/add-tinkerpop-modern.json | 121 ++++++++++++++++++ .../proxy-example/README.md | 23 ++++ .../proxy-example/compose.yaml | 14 +- .../proxy-example/conf/map/store.properties | 19 +++ .../conf/proxy/gafferpop.properties} | 11 +- .../conf/proxy}/operationsDeclarations.json | 0 .../conf/proxy}/store.properties | 4 +- .../proxy-example/conf}/schema/elements.json | 0 .../proxy-example/conf/schema/types.json | 43 +++++++ .../proxy-example}/create-deployment.sh | 4 +- .../operations/add-tinkerpop-modern.json} | 0 docker/gaffer-gremlin/Dockerfile | 28 ---- docker/gaffer-gremlin/README.md | 64 --------- docker/gaffer-gremlin/build.sh | 39 ------ .../conf/gaffer-gremlin-server.yaml | 70 ---------- .../conf/gafferpop/gafferpop.properties | 20 --- docker/gaffer-gremlin/pom.xml | 46 ------- docker/gaffer-rest/README.md | 12 -- .../config/schema/elements.json | 36 ------ .../config/schema/types.json | 28 ---- docker/gaffer-rest/proxy-example/README.md | 21 --- 47 files changed, 444 insertions(+), 444 deletions(-) create mode 100644 docker/example-deployments/README.md rename docker/{gaffer-gremlin/example/notebooks => example-deployments/example-notebooks}/graphnotebook-modern-example.ipynb (99%) rename docker/{gaffer-gremlin/example/notebooks => example-deployments/example-notebooks}/gremlinpython-modern-example.ipynb (81%) rename docker/{gaffer-rest => example-deployments}/federated-example/README.md (100%) rename docker/{gaffer-rest => example-deployments}/federated-example/compose.yaml (67%) rename docker/{gaffer-gremlin/compose.yaml => example-deployments/federated-example/conf/gaffer/gafferpop.properties} (71%) rename docker/{gaffer-rest/federated-example/config => example-deployments/federated-example/conf/gaffer}/store/defaultGraphIds.json (100%) rename docker/{gaffer-rest/federated-example/config => example-deployments/federated-example/conf/gaffer}/store/mergeFunctions.json (100%) rename docker/{gaffer-rest/federated-example/config => example-deployments/federated-example/conf/gaffer}/store/store.properties (100%) rename docker/{gaffer-rest => example-deployments}/federated-example/create-deployment.sh (100%) rename docker/{gaffer-rest => example-deployments}/federated-example/operations/add-modern-created-graph.json (100%) rename docker/{gaffer-rest => example-deployments}/federated-example/operations/add-modern-knows-graph.json (100%) create mode 100644 docker/example-deployments/modern-example/README.md rename docker/{gaffer-gremlin/example => example-deployments/modern-example}/compose.yaml (92%) rename docker/{gaffer-gremlin/conf/gafferpop => example-deployments/modern-example/conf/gaffer}/gafferpop.properties (70%) create mode 100644 docker/example-deployments/modern-example/conf/gaffer/schema/elements.json rename docker/{gaffer-gremlin/example => example-deployments/modern-example}/conf/gaffer/schema/types.json (100%) rename docker/{gaffer-gremlin/example => example-deployments/modern-example}/conf/gaffer/store-accumulo.properties (96%) create mode 100755 docker/example-deployments/modern-example/create-deployment.sh create mode 100644 docker/example-deployments/modern-example/operations/add-tinkerpop-modern.json create mode 100644 docker/example-deployments/proxy-example/README.md rename docker/{gaffer-rest => example-deployments}/proxy-example/compose.yaml (68%) create mode 100644 docker/example-deployments/proxy-example/conf/map/store.properties rename docker/{gaffer-gremlin/conf/gaffer/store.properties => example-deployments/proxy-example/conf/proxy/gafferpop.properties} (70%) rename docker/{gaffer-rest/proxy-example/config => example-deployments/proxy-example/conf/proxy}/operationsDeclarations.json (100%) rename docker/{gaffer-rest/proxy-example/config => example-deployments/proxy-example/conf/proxy}/store.properties (95%) rename docker/{gaffer-gremlin/example/conf/gaffer => example-deployments/proxy-example/conf}/schema/elements.json (100%) create mode 100644 docker/example-deployments/proxy-example/conf/schema/types.json rename docker/{gaffer-gremlin/example => example-deployments/proxy-example}/create-deployment.sh (94%) rename docker/{gaffer-gremlin/example/conf/gaffer/data/tinkerpop-modern.json => example-deployments/proxy-example/operations/add-tinkerpop-modern.json} (100%) delete mode 100644 docker/gaffer-gremlin/Dockerfile delete mode 100644 docker/gaffer-gremlin/README.md delete mode 100755 docker/gaffer-gremlin/build.sh delete mode 100644 docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml delete mode 100644 docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties delete mode 100644 docker/gaffer-gremlin/pom.xml delete mode 100644 docker/gaffer-rest/federated-example/config/schema/elements.json delete mode 100644 docker/gaffer-rest/federated-example/config/schema/types.json delete mode 100644 docker/gaffer-rest/proxy-example/README.md diff --git a/README.md b/README.md index 085a6be0..ad328977 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,29 @@ # Gaffer Docker This repo contains the code needed to run Gaffer using Docker or Kubernetes. -There are two main sub-folders, 'docker' and 'kubernetes' which contain the project files you need for starting Gaffer using those services. +There are two main sub-folders, 'docker' and 'kubernetes', which contain the +project files you need for starting Gaffer using those services. -# Running Gaffer Using Docker +## Running Gaffer Using Docker -For information on how to run Gaffer using Docker containers, please see the documentation: [Gaffer Docker Docs](https://gchq.github.io/gaffer-doc/latest/dev/docker/) +For information on how to run Gaffer using Docker containers please see the +documentation: [Gaffer Docker Docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gaffer-docker/gaffer-images.html) -# Running Gaffer Using Kubernetes +We also provide some example deployments with different store backings +to help you get started learning and testing Gaffer. Please see the +[example deployments](./docker/example-deployments/) directory for more +details. -For information on how to run Gaffer using Kubernetes, please see the documentation: [Gaffer Kubernetes Docs](https://gchq.github.io/gaffer-doc/latest/dev/kubernetes-guide/kubernetes/) +## Running Gaffer Using Kubernetes -# Versioning +For information on how to run Gaffer using Kubernetes, please see the +documentation: [Gaffer Kubernetes Docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/kubernetes-guide/running-on-kubernetes.html) -Each of our images which is released will be tagged with the version of the software they represent. Every release, -we update the `latest` tag for each image and add a new release which has the corresponding version tag. +## Versioning + +Each of the released images will be tagged with the version of the +software they represent. Every release we update the `latest` tag for each +image and add a new release which has the corresponding version tag. If we release Gaffer version 2.1.2, the following images would be uploaded: @@ -24,17 +33,20 @@ If we release Gaffer version 2.1.2, the following images would be uploaded: - gchq/gaffer:2.1.2 - gchq/gaffer:2.1.2-accumulo-2.0.1 -We maintain mutable versions of latest, as well as the major, minor and bugfix versions of Gaffer. For reproducibility -make sure to use the full version in your build metadata. For `gaffer`/`gaffer-rest` images, we also create a tag including the -accumulo version, this allows for compatibility with Accumulo 1.9.3 in our tests. The `-accumulo-1.9.3` tagged images -are not published but can be build locally if required. +We maintain mutable versions of latest, as well as the major, minor and bugfix +versions of Gaffer. For reproducibility make sure to use the full version in +your build metadata. For `gaffer`/`gaffer-rest` images, we also create a tag +including the accumulo version, this allows for compatibility with Accumulo +1.9.3 in our tests. The `-accumulo-1.9.3` tagged images are not published but +can be built locally if required. The release process is automated by GitHub actions. -# Known Compatible Docker Versions +## Known Compatible Docker Versions - 20.10.23 -# Contributing +## Contributing -We welcome contributions to this project. Detailed information on our ways of working can be found in our [developer docs](https://gchq.github.io/gaffer-doc/latest/dev/ways-of-working/). +We welcome contributions to this project. Detailed information on our ways of +working can be found in our [developer docs](https://gchq.github.io/gaffer-doc/latest/development-guide/ways-of-working.html). diff --git a/cd/build_core_images.sh b/cd/build_core_images.sh index 40929810..0a581120 100755 --- a/cd/build_core_images.sh +++ b/cd/build_core_images.sh @@ -26,7 +26,6 @@ pushd "${ROOT_DIR}" || exit 1 # HADOOP_VERSION # GAFFER_VERSION # ACCUMULO_VERSION -# TINKERPOP_VERSION if [[ -f "${1}" ]]; then source "${1}" else diff --git a/cd/build_images.sh b/cd/build_images.sh index f69b9ab5..7a4ea054 100755 --- a/cd/build_images.sh +++ b/cd/build_images.sh @@ -28,7 +28,6 @@ pushd "${ROOT_DIR}" || exit 1 # GAFFERPY_VERSION # ACCUMULO_VERSION # SPARK_VERSION -# TINKERPOP_VERSION if [[ -f "${1}" ]]; then source "${1}" else @@ -42,8 +41,6 @@ docker compose --project-directory ./docker/gaffer-road-traffic-loader/ -f ./doc # Builds all of the notebook related images: docker compose --project-directory ./docker/gaffer-pyspark-notebook/ -f ./docker/gaffer-pyspark-notebook/docker-compose.yaml build notebook docker compose --project-directory ./docker/spark-py/ -f ./docker/spark-py/docker-compose.yaml build -# Builds the Gaffer Gremlin server -./docker/gaffer-gremlin/build.sh # Set $JHUB_OPTIONS_SERVER_VERSION source ./docker/gaffer-jhub-options-server/get-version.sh diff --git a/cd/publish_images.sh b/cd/publish_images.sh index 5c017f1b..1f9a0d43 100755 --- a/cd/publish_images.sh +++ b/cd/publish_images.sh @@ -102,7 +102,6 @@ pushContainer gchq/accumulo "${ACCUMULO_VERSION}" pushContainer gchq/gaffer "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}" pushContainer gchq/gaffer-rest "${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}" pushContainer gchq/gaffer-road-traffic-loader "${GAFFER_VERSION}" -pushContainer gchq/gaffer-gremlin "${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION}" pushContainer gchq/gaffer-pyspark-notebook "${GAFFER_VERSION}" pushContainer gchq/gaffer-jhub-options-server "${JHUB_OPTIONS_SERVER_VERSION}" pushContainer gchq/spark-py "${SPARK_VERSION}" diff --git a/cd/update_versions.sh b/cd/update_versions.sh index dc15efcb..426bfe10 100755 --- a/cd/update_versions.sh +++ b/cd/update_versions.sh @@ -31,7 +31,6 @@ source ./docker/gaffer-jhub-options-server/get-version.sh find . -type f -exec sed -i "s/GAFFER_VERSION=[0-9]\.[0-9]\.[0-9]/GAFFER_VERSION=${APP_VERSION}/g" {} + find . -type f -exec sed -E -i "s/GAFFERPY_VERSION=(gafferpy-)?[0-9]\.[0-9]\.[0-9]/GAFFERPY_VERSION=\1${APP_VERSION}/g" {} + find . -type f -exec sed -i "s/GAFFER_TESTER_VERSION=[0-9]\.[0-9]\.[0-9]/GAFFER_TESTER_VERSION=${APP_VERSION}/g" {} + -sed -i'' -e "s/[0-9]\.[0-9]\.[0-9]/${APP_VERSION}/g" docker/gaffer-gremlin/pom.xml sed -i'' -e "s/BASE_IMAGE_TAG=[0-9]\.[0-9]\.[0-9]/BASE_IMAGE_TAG=${APP_VERSION}/g" docker/gaffer-kerberos/gaffer-krb/Dockerfile sed -i'' -e "s/BASE_IMAGE_TAG=[0-9]\.[0-9]\.[0-9]/BASE_IMAGE_TAG=${APP_VERSION}/g" docker/gaffer-kerberos/gaffer-rest-krb/Dockerfile diff --git a/docker/accumulo1.env b/docker/accumulo1.env index dacc4542..26f73892 100644 --- a/docker/accumulo1.env +++ b/docker/accumulo1.env @@ -7,5 +7,4 @@ GAFFER_VERSION=2.3.0 GAFFER_TESTER_VERSION=2.3.0 GAFFERPY_VERSION=2.3.0 SPARK_VERSION=3.1.2 -TINKERPOP_VERSION=3.7.1 KUBECTL_VERSION=1.23.0 diff --git a/docker/accumulo2.env b/docker/accumulo2.env index bea7dbcc..40ee1145 100644 --- a/docker/accumulo2.env +++ b/docker/accumulo2.env @@ -7,5 +7,4 @@ GAFFER_VERSION=2.3.0 GAFFER_TESTER_VERSION=2.3.0 GAFFERPY_VERSION=2.3.0 SPARK_VERSION=3.1.2 -TINKERPOP_VERSION=3.7.1 KUBECTL_VERSION=1.23.0 diff --git a/docker/example-deployments/README.md b/docker/example-deployments/README.md new file mode 100644 index 00000000..e11f8e0d --- /dev/null +++ b/docker/example-deployments/README.md @@ -0,0 +1,25 @@ +# Example Gaffer Deployments + +These directories have basic examples of deploying containerised Gaffer with +different store backings. Some example notebooks are also available which +primarily feature the use of the Gremlin interface for interacting with the +Graph. + +All examples will provide a Gaffer REST API to start interacting with the +deployed graph. + +## Modern Example (Accumulo) + +The modern example, which uses the [Tinkerpop modern dataset](https://tinkerpop.apache.org/docs/current/tutorials/the-gremlin-console/#toy-graphs), +can be found under the `modern-example` directory which features an Accumulo +store as its storage backing. + +## Federation Example + +A demo/example of using Gaffer REST with a Federated Store is available under +the `federated-example` directory. + +## Proxy Example + +A demo/example of using Gaffer REST with a Proxy Store is available under the +`proxy-example` directory. diff --git a/docker/gaffer-gremlin/example/notebooks/graphnotebook-modern-example.ipynb b/docker/example-deployments/example-notebooks/graphnotebook-modern-example.ipynb similarity index 99% rename from docker/gaffer-gremlin/example/notebooks/graphnotebook-modern-example.ipynb rename to docker/example-deployments/example-notebooks/graphnotebook-modern-example.ipynb index c23f2fb7..9ce6dffa 100644 --- a/docker/gaffer-gremlin/example/notebooks/graphnotebook-modern-example.ipynb +++ b/docker/example-deployments/example-notebooks/graphnotebook-modern-example.ipynb @@ -38,7 +38,7 @@ "%%graph_notebook_config\n", "{\n", " \"host\": \"localhost\",\n", - " \"port\": 8182,\n", + " \"port\": 8080,\n", " \"ssl\": false,\n", " \"gremlin\": {\n", " \"traversal_source\": \"g\",\n", diff --git a/docker/gaffer-gremlin/example/notebooks/gremlinpython-modern-example.ipynb b/docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb similarity index 81% rename from docker/gaffer-gremlin/example/notebooks/gremlinpython-modern-example.ipynb rename to docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb index 88a9913a..29ba70a0 100644 --- a/docker/gaffer-gremlin/example/notebooks/gremlinpython-modern-example.ipynb +++ b/docker/example-deployments/example-notebooks/gremlinpython-modern-example.ipynb @@ -21,10 +21,10 @@ "# Common imports\n", "from gremlin_python.process.anonymous_traversal import traversal\n", "from gremlin_python.process.graph_traversal import __\n", - "from gremlin_python.process.strategies import *\n", - "from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection\n", "from gremlin_python.process.traversal import *\n", + "from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection\n", "from gremlin_python.driver.client import Client\n", + "from gremlin_python.driver.serializer import GraphSONSerializersV3d0\n", "import nest_asyncio\n", "\n", "from_ = Direction.OUT\n", @@ -45,7 +45,11 @@ "metadata": {}, "outputs": [], "source": [ - "g = traversal().with_remote(DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))" + "# Traversal root\n", + "g = traversal().with_remote(DriverRemoteConnection('ws://localhost:8080/gremlin', 'g', message_serializer=GraphSONSerializersV3d0()))\n", + "\n", + "# Client connection\n", + "client = Client('ws://localhost:8080/gremlin', 'g', message_serializer=GraphSONSerializersV3d0())" ] }, { @@ -77,7 +81,7 @@ "metadata": {}, "outputs": [], "source": [ - "g.V('1', '2').hasLabel('person').outE('knows').values().is_(lt(1)).to_list()" + "g.V('1', '2').hasLabel('person').outE('knows').values().is_(P.lt(1)).to_list()" ] }, { @@ -96,13 +100,21 @@ "g.V('1').repeat(__.both().simplePath()).until(__.has('name', 'lop')).limit(3).path().by(__.valueMap()).to_list()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run a seeded cypher query via Gremlin. Submitted as a script rather than via gremlin-python bindings" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "g.V('1').outE().as_(\"a\").V('1').values(\"name\").to_list()" + "results = client.submit(\"g.with(\\\"cypher\\\", \\\"MATCH (p:person) WHERE ID(p) = '1' RETURN p\\\").call()\")\n", + "results.all().result()" ] } ], diff --git a/docker/gaffer-rest/federated-example/README.md b/docker/example-deployments/federated-example/README.md similarity index 100% rename from docker/gaffer-rest/federated-example/README.md rename to docker/example-deployments/federated-example/README.md diff --git a/docker/gaffer-rest/federated-example/compose.yaml b/docker/example-deployments/federated-example/compose.yaml similarity index 67% rename from docker/gaffer-rest/federated-example/compose.yaml rename to docker/example-deployments/federated-example/compose.yaml index d65cfdf9..f5e2512b 100644 --- a/docker/gaffer-rest/federated-example/compose.yaml +++ b/docker/example-deployments/federated-example/compose.yaml @@ -20,7 +20,8 @@ services: ports: - 8080:8080 volumes: - - ./config/store/store.properties:/gaffer/store/store.properties:ro - - ./config/store/mergeFunctions.json:/gaffer/store/mergeFunctions.json:ro - - ./config/store/defaultGraphIds.json:/gaffer/store/defaultGraphIds.json:ro - - ./config/schema:/gaffer/schema:ro + - ./conf/gaffer/schema:/gaffer/schema:ro + - ./conf/gaffer/store/store.properties:/gaffer/store/store.properties:ro + - ./conf/gaffer/store/mergeFunctions.json:/gaffer/store/mergeFunctions.json:ro + - ./conf/gaffer/store/defaultGraphIds.json:/gaffer/store/defaultGraphIds.json:ro + - ./conf/gaffer/gafferpop.properties:/gaffer/gafferpop.properties:ro diff --git a/docker/gaffer-gremlin/compose.yaml b/docker/example-deployments/federated-example/conf/gaffer/gafferpop.properties similarity index 71% rename from docker/gaffer-gremlin/compose.yaml rename to docker/example-deployments/federated-example/conf/gaffer/gafferpop.properties index ad8370f6..792dce8b 100644 --- a/docker/gaffer-gremlin/compose.yaml +++ b/docker/example-deployments/federated-example/conf/gaffer/gafferpop.properties @@ -1,3 +1,4 @@ +# # Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,15 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -version: "3.7" - -services: - gaffer-gremlin: - image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION} - build: - context: . - args: - BASE_IMAGE_TAG: ${TINKERPOP_VERSION} - ports: - - 8182:8182 +# +# Default operation config +gaffer.elements.getalllimit=5000 +gaffer.elements.hasstepfilterstage=PRE_AGGREGATION diff --git a/docker/gaffer-rest/federated-example/config/store/defaultGraphIds.json b/docker/example-deployments/federated-example/conf/gaffer/store/defaultGraphIds.json similarity index 100% rename from docker/gaffer-rest/federated-example/config/store/defaultGraphIds.json rename to docker/example-deployments/federated-example/conf/gaffer/store/defaultGraphIds.json diff --git a/docker/gaffer-rest/federated-example/config/store/mergeFunctions.json b/docker/example-deployments/federated-example/conf/gaffer/store/mergeFunctions.json similarity index 100% rename from docker/gaffer-rest/federated-example/config/store/mergeFunctions.json rename to docker/example-deployments/federated-example/conf/gaffer/store/mergeFunctions.json diff --git a/docker/gaffer-rest/federated-example/config/store/store.properties b/docker/example-deployments/federated-example/conf/gaffer/store/store.properties similarity index 100% rename from docker/gaffer-rest/federated-example/config/store/store.properties rename to docker/example-deployments/federated-example/conf/gaffer/store/store.properties diff --git a/docker/gaffer-rest/federated-example/create-deployment.sh b/docker/example-deployments/federated-example/create-deployment.sh similarity index 100% rename from docker/gaffer-rest/federated-example/create-deployment.sh rename to docker/example-deployments/federated-example/create-deployment.sh diff --git a/docker/gaffer-rest/federated-example/operations/add-modern-created-graph.json b/docker/example-deployments/federated-example/operations/add-modern-created-graph.json similarity index 100% rename from docker/gaffer-rest/federated-example/operations/add-modern-created-graph.json rename to docker/example-deployments/federated-example/operations/add-modern-created-graph.json diff --git a/docker/gaffer-rest/federated-example/operations/add-modern-knows-graph.json b/docker/example-deployments/federated-example/operations/add-modern-knows-graph.json similarity index 100% rename from docker/gaffer-rest/federated-example/operations/add-modern-knows-graph.json rename to docker/example-deployments/federated-example/operations/add-modern-knows-graph.json diff --git a/docker/example-deployments/modern-example/README.md b/docker/example-deployments/modern-example/README.md new file mode 100644 index 00000000..a96e35f4 --- /dev/null +++ b/docker/example-deployments/modern-example/README.md @@ -0,0 +1,26 @@ +# Gaffer Modern Example + +This is an example deployment of Gaffer on Accumulo. This uses a REST API +deployment of Gaffer with full Accumulo and Hadoop storage backing. + +The example uses the +[Tinkerpop Modern Graph](https://tinkerpop.apache.org/docs/current/tutorials/the-gremlin-console/#toy-graphs) +as its data set to demonstrate basic querying of data. The data can be queried +via the REST API using standard Gaffer operations or by using the Gremlin +websocket (see [the docs](https://gchq.github.io/gaffer-doc/latest/user-guide/query/gremlin/gremlin.html) +for more info). + +## Running the Example + +To run the example please use the provided start script with an environment file +to specify the image versions e.g. `accumulo2.env`: + +```bash +./create-deployment.sh ../../accumulo2.env +``` + +With the demo deployment a standard Gaffer REST API will be started (available +on port 8080 by default). Basic configuration for Accumulo and Hadoop have been +added but can be modified to test out different +scenarios, please see the [documentation](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-stores/accumulo-store.html) +for more information. diff --git a/docker/gaffer-gremlin/example/compose.yaml b/docker/example-deployments/modern-example/compose.yaml similarity index 92% rename from docker/gaffer-gremlin/example/compose.yaml rename to docker/example-deployments/modern-example/compose.yaml index 0c80c665..eb7a1f59 100644 --- a/docker/gaffer-gremlin/example/compose.yaml +++ b/docker/example-deployments/modern-example/compose.yaml @@ -173,16 +173,5 @@ services: volumes: - ./conf/gaffer/schema:/gaffer/schema:ro - ./conf/gaffer/store-accumulo.properties:/gaffer/store/store.properties:ro + - ./conf/gaffer/gafferpop.properties:/gaffer/gafferpop.properties:ro - # Gremlin server - gaffer-gremlin: - image: gchq/gaffer-gremlin:${GAFFER_VERSION}-gremlin-${TINKERPOP_VERSION} - depends_on: - accumulo-tserver: - condition: service_healthy - ports: - - 8182:8182 - volumes: - - ./conf/gafferpop:/opt/gremlin-server/conf/gafferpop:ro - - ./conf/gaffer/store-accumulo.properties:/opt/gremlin-server/conf/gaffer/store.properties:ro - - ./conf/gaffer/schema:/opt/gremlin-server/conf/gaffer/schema:ro diff --git a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties b/docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties similarity index 70% rename from docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties rename to docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties index e4e662b0..51c61b73 100644 --- a/docker/gaffer-gremlin/conf/gafferpop/gafferpop.properties +++ b/docker/example-deployments/modern-example/conf/gaffer/gafferpop.properties @@ -1,5 +1,4 @@ -# -# Copyright 2023-2024 Crown Copyright +# Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph -gaffer.graphId=graph1 -gaffer.storeproperties=conf/gaffer/store.properties -gaffer.schemas=conf/gaffer/schema/ -gaffer.userId=user01 +# Default operation config +gaffer.elements.getalllimit=5000 +gaffer.elements.hasstepfilterstage=PRE_AGGREGATION diff --git a/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json b/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json new file mode 100644 index 00000000..9764ec23 --- /dev/null +++ b/docker/example-deployments/modern-example/conf/gaffer/schema/elements.json @@ -0,0 +1,49 @@ +{ + "entities": { + "id": { + "vertex": "string.id" + }, + "person": { + "vertex": "string.id", + "properties": { + "name": "property.string", + "age": "property.integer" + } + }, + "software": { + "vertex": "string.id", + "properties": { + "name": "property.string", + "lang": "property.string" + } + } + }, + "edges": { + "knows": { + "source": "string.id", + "destination": "string.id", + "directed": "true", + "properties": { + "weight": "property.double" + } + }, + "created": { + "source": "string.id", + "destination": "string.id", + "directed": "true", + "properties": { + "weight": "property.double" + } + }, + "dependsOn": { + "source": "string.id", + "destination": "string.id", + "directed": "true" + }, + "encapsulates": { + "source": "string.id", + "destination": "string.id", + "directed": "true" + } + } +} diff --git a/docker/gaffer-gremlin/example/conf/gaffer/schema/types.json b/docker/example-deployments/modern-example/conf/gaffer/schema/types.json similarity index 100% rename from docker/gaffer-gremlin/example/conf/gaffer/schema/types.json rename to docker/example-deployments/modern-example/conf/gaffer/schema/types.json diff --git a/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties b/docker/example-deployments/modern-example/conf/gaffer/store-accumulo.properties similarity index 96% rename from docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties rename to docker/example-deployments/modern-example/conf/gaffer/store-accumulo.properties index e9de0f98..1a2ea5e5 100644 --- a/docker/gaffer-gremlin/example/conf/gaffer/store-accumulo.properties +++ b/docker/example-deployments/modern-example/conf/gaffer/store-accumulo.properties @@ -1,5 +1,5 @@ # -# Copyright 2023-2024 Crown Copyright +# Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docker/example-deployments/modern-example/create-deployment.sh b/docker/example-deployments/modern-example/create-deployment.sh new file mode 100755 index 00000000..df261c58 --- /dev/null +++ b/docker/example-deployments/modern-example/create-deployment.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CURRENT_DIR="$(dirname "$(readlink -f "${0}")")" +TINKERPOP_MODERN_JSON="${CURRENT_DIR}/operations/add-tinkerpop-modern.json" +GAFFER_REST_URL="http://localhost:8080" + +# Make sure we have an evironment file for the image versions +if [ ! -f "${1}" ]; then + echo "Error - Environment file not set" + echo "Example: ${0} .env" + exit 1 +fi + +# Remove any existing containers +docker compose --env-file "${1}" -f "${CURRENT_DIR}/compose.yaml" down + +# Start new deployment +echo "-----------------------------------" +echo "Starting new deployment..." +docker compose --env-file "${1}" -f "${CURRENT_DIR}/compose.yaml" up --detach + +# Check can connect to instance +echo "-----------------------------------" +echo "Attempting to connect to Gaffer instance..." +if ! docker run --rm --network=host curlimages/curl:latest \ + curl --retry 20 \ + --retry-delay 5 \ + --retry-all-errors "${GAFFER_REST_URL}"/rest/graph/status +then + echo "Failed to connect to Gaffer instance" + exit 1 +fi + +# Load dataset +echo "-----------------------------------" +echo "Loading data into Gaffer instance..." +if ! docker run --rm --network=host curlimages/curl:latest \ + curl -H "Content-Type: application/json" \ + --data "$(cat "${TINKERPOP_MODERN_JSON}")" "${GAFFER_REST_URL}"/rest/graph/operations/execute +then + echo "Failed to load tinkerpop modern JSON: ${TINKERPOP_MODERN_JSON}" + exit 1 +fi + +echo "-----------------------------------" +echo "Deployment started! REST API: ${GAFFER_REST_URL}" diff --git a/docker/example-deployments/modern-example/operations/add-tinkerpop-modern.json b/docker/example-deployments/modern-example/operations/add-tinkerpop-modern.json new file mode 100644 index 00000000..b6d5f8b7 --- /dev/null +++ b/docker/example-deployments/modern-example/operations/add-tinkerpop-modern.json @@ -0,0 +1,121 @@ +{ + "class": "AddElements", + "input": [ + { + "class": "Entity", + "group": "person", + "vertex": "1", + "properties": { + "name": "marko", + "age": 29 + } + }, + { + "class": "Entity", + "group": "person", + "vertex": "2", + "properties": { + "name": "vadas", + "age": 27 + } + }, + { + "class": "Entity", + "group": "person", + "vertex": "4", + "properties": { + "name": "josh", + "age": 32 + } + }, + { + "class": "Entity", + "group": "person", + "vertex": "6", + "properties": { + "name": "peter", + "age": 35 + } + }, + { + "class": "Entity", + "group": "software", + "vertex": "3", + "properties": { + "name": "lop", + "lang": "java" + } + }, + { + "class": "Entity", + "group": "software", + "vertex": "5", + "properties": { + "name": "ripple", + "lang": "java" + } + }, + { + "class": "Edge", + "group": "knows", + "source": "1", + "destination": "2", + "directed": true, + "properties": { + "weight": 0.5 + } + }, + { + "class": "Edge", + "group": "created", + "source": "1", + "destination": "3", + "directed": true, + "properties": { + "weight": 0.4 + } + }, + { + "class": "Edge", + "group": "knows", + "source": "1", + "destination": "4", + "directed": true, + "properties": { + "weight": 1.0 + } + }, + { + "class": "Edge", + "group": "created", + "source": "4", + "destination": "3", + "directed": true, + "properties": { + "weight": 0.4 + } + }, + { + "class": "Edge", + "group": "created", + "source": "4", + "destination": "5", + "directed": true, + "properties": { + "weight": 1.0 + } + }, + { + "class": "Edge", + "group": "created", + "source": "6", + "destination": "3", + "directed": true, + "properties": { + "weight": 0.2 + } + } + ], + "skipInvalidElements": false, + "validate": true +} diff --git a/docker/example-deployments/proxy-example/README.md b/docker/example-deployments/proxy-example/README.md new file mode 100644 index 00000000..57aa19ce --- /dev/null +++ b/docker/example-deployments/proxy-example/README.md @@ -0,0 +1,23 @@ +# Proxy Example + +## Running + +This is an example deployment of Gaffer using a Proxy store. This uses two REST +APIs, one contains the data and is backed by a Map Store and the other is a +Proxy Store configured to forward queries to the Map Store backed REST API. + +The example uses the +[Tinkerpop Modern Graph](https://tinkerpop.apache.org/docs/current/tutorials/the-gremlin-console/#toy-graphs) +as its data set to demonstrate basic querying of data. + +## Running the Example + +To run the example please use the provided start script with an environment file +to specify the image versions e.g. `accumulo2.env`: + +```bash +./create-deployment.sh ../../accumulo2.env +``` + +With the demo deployment the Proxy backed Gaffer REST API will be started on port 8080 +and the Map store backed instance on 8081. diff --git a/docker/gaffer-rest/proxy-example/compose.yaml b/docker/example-deployments/proxy-example/compose.yaml similarity index 68% rename from docker/gaffer-rest/proxy-example/compose.yaml rename to docker/example-deployments/proxy-example/compose.yaml index 19926e0d..811b3f84 100644 --- a/docker/gaffer-rest/proxy-example/compose.yaml +++ b/docker/example-deployments/proxy-example/compose.yaml @@ -19,7 +19,12 @@ services: healthcheck: test: sleep 15 timeout: 20s + ports: + - 8081:8080 restart: always + volumes: + - ./conf/map/store.properties:/gaffer/store/store.properties:ro + - ./conf/schema:/gaffer/schema:ro gaffer-rest-proxy: image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} @@ -27,8 +32,9 @@ services: gaffer-rest: condition: service_healthy restart: always - ports: - - 8080:8080 + network_mode: "host" volumes: - - ./config/store.properties:/gaffer/store/store.properties:ro - - ./config/operationsDeclarations.json:/gaffer/store/operationsDeclarations.json:ro + - ./conf/proxy/store.properties:/gaffer/store/store.properties:ro + - ./conf/proxy/operationsDeclarations.json:/gaffer/store/operationsDeclarations.json:ro + - ./conf/proxy/gafferpop.properties:/gaffer/gafferpop.properties:ro + - ./conf/schema:/gaffer/schema:ro diff --git a/docker/example-deployments/proxy-example/conf/map/store.properties b/docker/example-deployments/proxy-example/conf/map/store.properties new file mode 100644 index 00000000..4b0566f3 --- /dev/null +++ b/docker/example-deployments/proxy-example/conf/map/store.properties @@ -0,0 +1,19 @@ +# Copyright 2024 Crown Copyright +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +gaffer.store.class=uk.gov.gchq.gaffer.mapstore.MapStore +gaffer.store.properties.class=uk.gov.gchq.gaffer.mapstore.MapStoreProperties + +# General store config +gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService +gaffer.store.job.tracker.enabled=true diff --git a/docker/gaffer-gremlin/conf/gaffer/store.properties b/docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties similarity index 70% rename from docker/gaffer-gremlin/conf/gaffer/store.properties rename to docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties index 6dad3cd4..51c61b73 100644 --- a/docker/gaffer-gremlin/conf/gaffer/store.properties +++ b/docker/example-deployments/proxy-example/conf/proxy/gafferpop.properties @@ -1,5 +1,4 @@ -# -# Copyright 2023-2024 Crown Copyright +# Copyright 2024 Crown Copyright # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore -# These can be configured to an existing graph deployment -gaffer.host=localhost -gaffer.port=8080 -gaffer.context-root=/rest/latest +# Default operation config +gaffer.elements.getalllimit=5000 +gaffer.elements.hasstepfilterstage=PRE_AGGREGATION diff --git a/docker/gaffer-rest/proxy-example/config/operationsDeclarations.json b/docker/example-deployments/proxy-example/conf/proxy/operationsDeclarations.json similarity index 100% rename from docker/gaffer-rest/proxy-example/config/operationsDeclarations.json rename to docker/example-deployments/proxy-example/conf/proxy/operationsDeclarations.json diff --git a/docker/gaffer-rest/proxy-example/config/store.properties b/docker/example-deployments/proxy-example/conf/proxy/store.properties similarity index 95% rename from docker/gaffer-rest/proxy-example/config/store.properties rename to docker/example-deployments/proxy-example/conf/proxy/store.properties index 49bd9c5d..27271e01 100644 --- a/docker/gaffer-rest/proxy-example/config/store.properties +++ b/docker/example-deployments/proxy-example/conf/proxy/store.properties @@ -16,6 +16,6 @@ gaffer.store.class=uk.gov.gchq.gaffer.proxystore.ProxyStore gaffer.store.operation.declarations=/gaffer/store/operationsDeclarations.json # Proxy config -gaffer.host=gaffer-rest -gaffer.port=8080 +gaffer.host=localhost +gaffer.port=8081 gaffer.context-root=/rest diff --git a/docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json b/docker/example-deployments/proxy-example/conf/schema/elements.json similarity index 100% rename from docker/gaffer-gremlin/example/conf/gaffer/schema/elements.json rename to docker/example-deployments/proxy-example/conf/schema/elements.json diff --git a/docker/example-deployments/proxy-example/conf/schema/types.json b/docker/example-deployments/proxy-example/conf/schema/types.json new file mode 100644 index 00000000..2f632371 --- /dev/null +++ b/docker/example-deployments/proxy-example/conf/schema/types.json @@ -0,0 +1,43 @@ +{ + "types": { + "int.id": { + "class": "java.lang.Integer" + }, + "long.id": { + "class": "java.lang.Long" + }, + "string.id": { + "class": "java.lang.String" + }, + "uuid.id": { + "class": "java.util.UUID" + }, + "true": { + "class": "java.lang.Boolean" + }, + "property.string": { + "class": "java.lang.String", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + }, + "property.integer": { + "class": "java.lang.Integer", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + }, + "property.double": { + "class": "java.lang.Double", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + }, + "property.float": { + "class": "java.lang.Float", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" + } + } + } +} diff --git a/docker/gaffer-gremlin/example/create-deployment.sh b/docker/example-deployments/proxy-example/create-deployment.sh similarity index 94% rename from docker/gaffer-gremlin/example/create-deployment.sh rename to docker/example-deployments/proxy-example/create-deployment.sh index bf531a09..f0eff1d7 100755 --- a/docker/gaffer-gremlin/example/create-deployment.sh +++ b/docker/example-deployments/proxy-example/create-deployment.sh @@ -14,8 +14,8 @@ # limitations under the License. CURRENT_DIR="$(dirname "$(readlink -f "${0}")")" -TINKERPOP_MODERN_JSON="${CURRENT_DIR}/conf/gaffer/data/tinkerpop-modern.json" -GAFFER_REST_URL="http://localhost:8080" +TINKERPOP_MODERN_JSON="${CURRENT_DIR}/operations/add-tinkerpop-modern.json" +GAFFER_REST_URL="http://localhost:8081" # Make sure we have an evironment file for the image versions if [ ! -f "${1}" ]; then diff --git a/docker/gaffer-gremlin/example/conf/gaffer/data/tinkerpop-modern.json b/docker/example-deployments/proxy-example/operations/add-tinkerpop-modern.json similarity index 100% rename from docker/gaffer-gremlin/example/conf/gaffer/data/tinkerpop-modern.json rename to docker/example-deployments/proxy-example/operations/add-tinkerpop-modern.json diff --git a/docker/gaffer-gremlin/Dockerfile b/docker/gaffer-gremlin/Dockerfile deleted file mode 100644 index 5fa93c14..00000000 --- a/docker/gaffer-gremlin/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023-2024 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG BASE_IMAGE_NAME=tinkerpop/gremlin-server -ARG BASE_IMAGE_TAG=3.7.1 - -# Base image provides WORKDIR, USER and ENTRYPOINT -FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} - -# Copy JARS -COPY ./target/dependency/*.jar ext/gafferpop/plugin/ - -# Copy configs -COPY ./conf/ conf/ - -# Set server to run -CMD ["conf/gaffer-gremlin-server.yaml"] diff --git a/docker/gaffer-gremlin/README.md b/docker/gaffer-gremlin/README.md deleted file mode 100644 index c6621342..00000000 --- a/docker/gaffer-gremlin/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# gaffer-gremlin - -## Build - -Run the supplied `build.sh` script with the required environment vars (can -source `accumolo2.env` file) to pull all dependencies and build the container. - -_Note requires Maven and Docker_ - -Optionally run each command separately to configure the containers tags etc. - -## Configure - -The container will use the Gaffer Proxy store by default to connect to an -existing graph and provide a Gremlin endpoint to connect to (see the [Tinkerpop docs](https://tinkerpop.apache.org/docs/current/reference/#connecting-gremlin-server)). - -The Gaffer graph the container will connect to can be configured as usual by -editing the `store.properties` file, which you can also bind mount over on an -existing image. The config file locations are under the predefined workdir -set by the parent gremlin server image the key locations in the image are: - -- `/opt/gremlin-server/conf/gaffer/store.properties` - Override for custom store properties. -- `/opt/gremlin-server/conf/gafferpop/gafferpop.properties` - Override to configure the graph. - -The configuration for the Gremlin server is provided by the `gaffer-gremlin-server.yaml` -this again can be modified as needed or bind mounted over. Please see the -[official Gaffer docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-deployment/gremlin/) -for more information on configuring this image. - -## Run - -Simply run the container to publish the Gremlin server making sure to make the -configured port available (port 8182 by default) this can then be connected to -via the address specified in the server's yaml config to use Gremlin traversal. -The server can then be connected via the Gremlin console e.g. if using -`gremlinpython`: - -```python -from gremlin_python.process.anonymous_traversal_source import traversal - -g = traversal().withRemote( - DriverRemoteConnection('ws://localhost:8182/gremlin', 'g')) -``` - -### Demo Deployment - -A demo/example using the tinkerpop 'modern' dataset and accumulo backed Gaffer -is available under the `example` directory. This can be ran using docker compose -to deploy the containers. The provided jupyter notebooks demonstrate how to -connect to your graph and some basic Gremlin queries on the data using `gremlinpython` or via -`graph-notebook` (note this requires the [graph-notebook extension](https://github.com/aws/graph-notebook)). - -To run the example please use the provided start script with an environment file -to specify the image versions e.g. `accumulo2.env`: - -```bash -./example/create-deployment.sh ../accumulo2.env -``` - -With the demo deployment both a standard Gaffer REST API and Gaffer Gremlin -server will be started (available on port 8080 and 8182 respectively). The -deployment uses a small scale Accumulo instance as the Gaffer store backing, -see [the docs](https://gchq.github.io/gaffer-doc/latest/administration-guide/gaffer-stores/accumulo-store/) -for more configuration options. diff --git a/docker/gaffer-gremlin/build.sh b/docker/gaffer-gremlin/build.sh deleted file mode 100755 index a54ef961..00000000 --- a/docker/gaffer-gremlin/build.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright 2023-2024 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -declare SCRIPT_DIR="$(readlink -f "$(dirname "${0}")")" - -# Check if already have env var for version -[[ -z "${GAFFER_VERSION}" ]] && \ - echo "Missing GAFFER_VERSION env var" && \ - exit 1 -[[ -z "${TINKERPOP_VERSION}" ]] && \ - echo "Missing TINKERPOP_VERSION env var" && \ - exit 1 - -# Build from relevant directory -pushd "${SCRIPT_DIR}" || exit 1 - # Download JARs - mvn clean dependency:copy-dependencies \ - --define gaffer.version="${GAFFER_VERSION}" - - # Build container - docker compose build - - # Clean - mvn clean -popd || exit 1 - -echo "Build Successful" diff --git a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml b/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml deleted file mode 100644 index 45f8714d..00000000 --- a/docker/gaffer-gremlin/conf/gaffer-gremlin-server.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2023-2024 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -host: localhost -port: 8182 -evaluationTimeout: 30000 -graphs: { - graph: conf/gafferpop/gafferpop.properties -} -scriptEngines: { - gremlin-groovy: { - plugins: { - uk.gov.gchq.gaffer.tinkerpop.gremlinplugin.GafferPopGremlinPlugin: {}, - org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, - org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyCompilerGremlinPlugin: { - enableThreadInterrupt: true - }, - org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { - classImports: [java.lang.Math], - methodImports: [java.lang.Math#*] - }, - org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { - files: [scripts/empty-sample.groovy] - } - } - } -} - -serializers: - # application/vnd.graphbinary-v1.0 - - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1 } - # application/vnd.graphbinary-v1.0-stringd - - { className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} - # application/json - - { className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3, config: { ioRegistries: [org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerIoRegistryV3] }} - -# Uncomment following sections to configure the user authentication to the server -# # This should be a deployment specific class -# authentication: { -# authenticator: uk.gov.gchq.gaffer.tinkerpop.server.auth.ExampleGafferPopAuthenticator -# } -# # This class is necessary for correctly forwarding the user to Gaffer -# authorization: { -# authorizer: uk.gov.gchq.gaffer.tinkerpop.server.auth.GafferPopAuthoriser -# } - -metrics: { - slf4jReporter: {enabled: true, interval: 180000} -} -strictTransactionManagement: false -idleConnectionTimeout: 0 -keepAliveInterval: 0 -maxInitialLineLength: 4096 -maxHeaderSize: 8192 -maxChunkSize: 8192 -maxContentLength: 10485760 -maxAccumulationBufferComponents: 1024 -resultIterationBatchSize: 64 -enableAuditLog: true diff --git a/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties b/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties deleted file mode 100644 index 4a4920a2..00000000 --- a/docker/gaffer-gremlin/example/conf/gafferpop/gafferpop.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2023-2024 Crown Copyright -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -gremlin.graph=uk.gov.gchq.gaffer.tinkerpop.GafferPopGraph -gaffer.graphId=superGraph -gaffer.storeproperties=conf/gaffer/store.properties -gaffer.schemas=conf/gaffer/schema/ -gaffer.userId=user01 diff --git a/docker/gaffer-gremlin/pom.xml b/docker/gaffer-gremlin/pom.xml deleted file mode 100644 index 50e3b706..00000000 --- a/docker/gaffer-gremlin/pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - 4.0.0 - - uk.gov.gchq.gaffer.docker - gaffer-gremlin - 0.0.1-SNAPSHOT - pom - - - - 2.3.0 - 2.36 - - - - - - uk.gov.gchq.gaffer - tinkerpop - ${gaffer.version} - - - - org.glassfish.jersey.inject - jersey-hk2 - ${jersey.version} - - - - diff --git a/docker/gaffer-rest/README.md b/docker/gaffer-rest/README.md index 06baa6d2..5ee9e645 100644 --- a/docker/gaffer-rest/README.md +++ b/docker/gaffer-rest/README.md @@ -70,15 +70,3 @@ directory. These will be added to the classpath at runtime. * Gaffer REST Access the Gaffer REST API at: `http://localhost:8080/rest/` - -## Examples - -### Proxy Example - -A demo/example of using Gaffer REST with a Proxy Store is available under the -`proxy-example` directory. - -### Federation Example - -A demo/example of using Gaffer REST with a Federated Store is available under -the `federated-example` directory. diff --git a/docker/gaffer-rest/federated-example/config/schema/elements.json b/docker/gaffer-rest/federated-example/config/schema/elements.json deleted file mode 100644 index 3795106b..00000000 --- a/docker/gaffer-rest/federated-example/config/schema/elements.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "entities": { - "person": { - "vertex": "string.id", - "properties": { - "name": "property.string", - "age": "property.integer" - } - }, - "software": { - "vertex": "string.id", - "properties": { - "name": "property.string", - "lang": "property.string" - } - } - }, - "edges": { - "knows": { - "source": "string.id", - "destination": "string.id", - "directed": "true", - "properties": { - "weight": "property.double" - } - }, - "created": { - "source": "string.id", - "destination": "string.id", - "directed": "true", - "properties": { - "weight": "property.double" - } - } - } -} diff --git a/docker/gaffer-rest/federated-example/config/schema/types.json b/docker/gaffer-rest/federated-example/config/schema/types.json deleted file mode 100644 index 5550cea5..00000000 --- a/docker/gaffer-rest/federated-example/config/schema/types.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "types": { - "string.id": { - "class": "java.lang.String" - }, - "true": { - "class": "java.lang.Boolean" - }, - "property.string": { - "class": "java.lang.String", - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" - } - }, - "property.integer": { - "class": "java.lang.Integer", - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" - } - }, - "property.double": { - "class": "java.lang.Double", - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.First" - } - } - } -} diff --git a/docker/gaffer-rest/proxy-example/README.md b/docker/gaffer-rest/proxy-example/README.md deleted file mode 100644 index 627fbcf4..00000000 --- a/docker/gaffer-rest/proxy-example/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Proxy Example - -## Running - -To run it from the `proxy-example` directory you'll need to specify the -environment file path: - -```bash -docker compose --env-file ../.env up -``` - -Or to run it from this directory, the compose file path and environment file -path: - -```bash -docker compose -f proxy-example/compose.yaml --env-file .env up -``` - -With this deployment both a standard Gaffer REST API and a Gaffer REST API using -the Proxy Store will be started. Only the port for the Proxy REST endpoint is -exposed, this will forward operations to the standard Gaffer REST Endpoint.