Merge branch 'develop' into gh-334-smoother-gaffer-gremlin-deploy

.github/workflows/continuous-integration.yaml

@@ -27,10 +27,14 @@ jobs:
  steps:
 
  - name: Maximize build space
- uses: easimon/maximize-build-space@v4
+ uses: easimon/maximize-build-space@v10
  with:
  remove-dotnet: 'true'
  remove-android: 'true'
+ remove-haskell: 'true'
+ remove-codeql: 'true'
+ remove-docker-images: 'true'
+ swap-size-mb: 1024
  overprovision-lvm: 'true'
 
  - uses: actions/checkout@v4

docker/accumulo1.env

@@ -3,7 +3,7 @@ HADOOP_VERSION=3.2.2
 ACCUMULO_VERSION=1.9.3
 HADOOP_CONF_DIR=/etc/hadoop/conf
 ACCUMULO_CONF_DIR=/etc/accumulo/conf
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
 SPARK_VERSION=3.1.2

docker/gaffer-federated-integration-tests/.env

@@ -1,6 +1,6 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-integration-tests/.env

@@ -1,6 +1,6 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GAFFER_TESTER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFER_TESTER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-kerberos/.env

@@ -1,4 +1,4 @@
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 GAFFER_KRB_PASSWORD=change-password-for-gaffer
 
 ZOOKEEPER_VERSION=3.7.1

docker/gaffer-kerberos/gaffer-integration-tests-krb/.env

@@ -1,6 +1,6 @@
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 GAFFER_KRB_PASSWORD=change-password-for-gaffer
-GAFFER_TESTER_VERSION=2.0.0
+GAFFER_TESTER_VERSION=2.1.0
 
 ZOOKEEPER_VERSION=3.7.1
 ZOOKEEPER_KRB_PASSWORD=change-password-for-zookeeper

docker/gaffer-kerberos/gaffer-krb/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE_NAME=gchq/gaffer
-ARG BASE_IMAGE_TAG=2.0.0
+ARG BASE_IMAGE_TAG=2.1.0
 
 ARG USER=accumulo
 ARG GROUP=accumulo

docker/gaffer-kerberos/gaffer-rest-krb/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2023 Crown Copyright
+# Copyright 2023-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE_NAME=gchq/gaffer-rest
-ARG BASE_IMAGE_TAG=2.0.0
+ARG BASE_IMAGE_TAG=2.1.0
 
 FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
 

docker/gaffer-pyspark-notebook/.env

@@ -1,6 +1,6 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
-GAFFERPY_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
+GAFFERPY_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 SPARK_VERSION=3.1.2

docker/gaffer-pyspark-notebook/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -60,7 +60,7 @@ ARG KUBECTL_VERSION=1.23.0
 RUN curl -fLo /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
  chmod +x /bin/kubectl
 
-ARG GAFFERPY_VERSION=gafferpy-2.0.0
+ARG GAFFERPY_VERSION=gafferpy-2.1.0
 RUN git clone -b ${GAFFERPY_VERSION} --depth 1 https://github.com/gchq/gafferpy && \
  pushd gafferpy && \
  python setup.py install && \

docker/gaffer-rest/.env

@@ -1,2 +1,2 @@
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1

docker/gaffer-rest/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ ARG BASE_IMAGE_TAG=8-jre-alpine
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
 ARG ACCUMULO_VERSION=2.0.1
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 

docker/gaffer-road-traffic-loader/.env

@@ -1,5 +1,5 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer-road-traffic-loader/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ ARG BASE_IMAGE_TAG=8u322-jre-slim-bullseye
 
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 

docker/gaffer-road-traffic-loader/config/schema/types.json

@@ -100,12 +100,12 @@
  }
  },
  "hllp": {
- "class": "com.clearspring.analytics.stream.cardinality.HyperLogLogPlus",
+ "class": "org.apache.datasketches.hll.HllSketch",
  "aggregateFunction": {
- "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.binaryoperator.HyperLogLogPlusAggregator"
+ "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator"
  },
  "serialiser": {
- "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.serialisation.HyperLogLogPlusSerialiser"
+ "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser"
  }
  },
  "counts.freqmap": {

docker/gaffer/.env

@@ -1,5 +1,5 @@
 ZOOKEEPER_VERSION=3.7.1
-GAFFER_VERSION=2.0.0
+GAFFER_VERSION=2.1.0
 ACCUMULO_VERSION=2.0.1
 HADOOP_VERSION=3.3.3
 ACCUMULO_CONF_DIR=/etc/accumulo/conf

docker/gaffer/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ ARG BASE_IMAGE_TAG=2.0.1
 FROM ${BUILDER_IMAGE_NAME}:${BUILDER_IMAGE_TAG} as builder
 
 ARG BASE_IMAGE_TAG
-ARG GAFFER_VERSION=2.0.0
+ARG GAFFER_VERSION=2.1.0
 ARG GAFFER_LIBS=bitmap-library,sketches-library,time-library
 ARG GAFFER_DOWNLOAD_URL=https://repo1.maven.org/maven2
 ARG GAFFER_GIT_REPO=https://github.com/gchq/Gaffer.git
@@ -60,5 +60,9 @@ RUN allFilesDownloaded="TRUE" && \
  done \
  fi
 
+# Hotfix for Gaffer 2.1.0, see Gaffer issue #3159
+RUN wget -nv "${GAFFER_DOWNLOAD_URL}/org/apache/datasketches/datasketches-memory/2.2.0/datasketches-memory-2.2.0.jar"
+RUN wget -nv "${GAFFER_DOWNLOAD_URL}/org/apache/datasketches/datasketches-java/4.0.0/datasketches-java-4.0.0.jar"
+
 FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
 COPY --from=builder --chown=root:root /jars/*.jar /opt/accumulo/lib/ext/

kubernetes/accumulo/Chart.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@ apiVersion: v2
 name: accumulo
 description: A Key-Value Accumulo store
 type: application
-version: 2.0.0 # managed version
+version: 2.1.0 # managed version
 appVersion: 2.0.1
 home: https://github.com/gchq/Gaffer
 sources:
@@ -26,6 +26,6 @@ dependencies:
  repository: https://charts.helm.sh/incubator/
  condition: zookeeper.enabled
  - name: hdfs
- version: ^2.0.0 # managed version
+ version: ^2.1.0 # managed version
  repository: file://../hdfs/
  condition: hdfs.enabled

kubernetes/gaffer-jhub/Chart.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@ apiVersion: v2
 name: gaffer-jhub
 description: A Jupyter Hub instance, with Gaffer integrations
 type: application
-version: 2.0.0 # managed version
+version: 2.1.0 # managed version
 home: https://github.com/gchq/Gaffer
 sources:
  - https://github.com/gchq/gaffer-docker

kubernetes/gaffer-jhub/values.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,7 +65,7 @@ jupyterhub:
  kubespawner_override:
  image: jupyter/minimal-notebook:399cbb986c6b
  - display_name: "Gaffer pySpark Notebook"
- description: "Python 3, Hadoop 3.3.3, Spark 3.1.2, AWS CLI 2, kubectl 1.23.0, gafferpy 2.0.0" # managed
+ description: "Python 3, Hadoop 3.3.3, Spark 3.1.2, AWS CLI 2, kubectl 1.23.0, gafferpy 2.1.0" # managed
  slug: "gaffer-pyspark"
  default: true
  enable_hdfs: true
@@ -74,7 +74,7 @@ jupyterhub:
  spark_image: gchq/spark-py:3.1.2 # managed
  spark_ingress_host: "{{USERNAME}}-{{SERVERNAME}}.spark.example.com"
  kubespawner_override:
- image: gchq/gaffer-pyspark-notebook:2.0.0 # managed
+ image: gchq/gaffer-pyspark-notebook:2.1.0 # managed
 optionsServer:
  replicaCount: 1
  image:
@@ -141,4 +141,4 @@ optionsServer:
 testImages:
  python:
  repository: gchq/gaffer-pyspark-notebook
- tag: 2.0.0 # managed version
+ tag: 2.1.0 # managed version

kubernetes/gaffer-road-traffic/Chart.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@ apiVersion: v2
 name: gaffer-road-traffic
 description: A Gaffer instance containing sample GB road traffic data from the Department of Transport
 type: application
-version: 2.0.0 # managed version
-appVersion: 2.0.0 # managed version
+version: 2.1.0 # managed version
+appVersion: 2.1.0 # managed version
 home: https://github.com/gchq/Gaffer
 sources:
  - https://github.com/gchq/gaffer-docker
  - https://data.gov.uk/dataset/208c0e7b-353f-4e2d-8b7a-1a7118467acc/gb-road-traffic-counts
 dependencies:
  - name: gaffer
- version: ^2.0.0 # managed version
+ version: ^2.1.0 # managed version
  repository: file://../gaffer/

kubernetes/gaffer-road-traffic/values.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ labels: {}
 loader:
  image:
  repository: gchq/gaffer-road-traffic-loader
- tag: 2.0.0 # managed version
+ tag: 2.1.0 # managed version
  pullPolicy: IfNotPresent
  pullSecrets: []
  resources: {}
@@ -206,12 +206,12 @@ gaffer:
  }
  },
  "hllp": {
- "class": "com.clearspring.analytics.stream.cardinality.HyperLogLogPlus",
+ "class": "org.apache.datasketches.hll.HllSketch",
  "aggregateFunction": {
- "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.binaryoperator.HyperLogLogPlusAggregator"
+ "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator"
  },
  "serialiser": {
- "class": "uk.gov.gchq.gaffer.sketches.clearspring.cardinality.serialisation.HyperLogLogPlusSerialiser"
+ "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser"
  }
  },
  "counts.freqmap": {

kubernetes/gaffer/Chart.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@ apiVersion: v2
 name: gaffer
 description: A large-scale entity and relation database supporting aggregation of properties
 type: application
-version: 2.0.0 # managed version
-appVersion: 2.0.0
+version: 2.1.0 # managed version
+appVersion: 2.1.0
 home: https://github.com/gchq/Gaffer
 sources:
  - https://github.com/gchq/gaffer-docker
 dependencies:
  - name: accumulo
- version: ^2.0.0 # managed version
+ version: ^2.1.0 # managed version
  repository: file://../accumulo/
  condition: accumulo.enabled

kubernetes/gaffer/templates/graph-status-check-hook.yaml

@@ -1,6 +1,6 @@
 {{- /*
 
-Copyright 2020 Crown Copyright
+Copyright 2020-2024 Crown Copyright
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -31,6 +31,6 @@ spec:
  containers:
  - name: curl
  image: {{ .Values.test.images.curl.repository }}:{{ .Values.test.images.curl.tag}}
- command: ["/bin/sh", "-c", "statusCode=-1 && until [ \"${statusCode}\" = \"200\" ] || [ $(( ATTEMPTS++ )) -gt 300 ]; do sleep 1; statusCode=$(curl -f -s -o /dev/null -w \"%{http_code}\" http://{{ include "gaffer.fullname" . }}-api:80/{{ pluck "server.context-path" .Values.graph.applicationProperties | first }}/graph/status); echo \"$(date) - http://{{ include "gaffer.fullname" . }}-api:80{{ pluck "server.context-path" .Values.graph.applicationProperties | first }}/graph/status : ${statusCode}\"; done; [ \"${statusCode}\" != \"200\" ] && exit 1; exit 0"]
+ command: ["/bin/sh", "-c", "statusCode=-1 && until [ \"${statusCode}\" = \"200\" ] || [ $(( ATTEMPTS++ )) -gt 300 ]; do sleep 1; statusCode=$(curl -f -s -o /dev/null -w \"%{http_code}\" http://{{ include "gaffer.fullname" . }}-api:80{{ pluck "server.context-path" .Values.graph.applicationProperties | first }}/graph/status); echo \"$(date) - http://{{ include "gaffer.fullname" . }}-api:80{{ pluck "server.context-path" .Values.graph.applicationProperties | first }}/graph/status : ${statusCode}\"; done; [ \"${statusCode}\" != \"200\" ] && exit 1; exit 0"]
  restartPolicy: Never
 {{- end }}

kubernetes/gaffer/values.yaml

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Crown Copyright
+# Copyright 2020-2024 Crown Copyright
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@ api:
  replicaCount: 1
  image:
  repository: gchq/gaffer-rest
- tag: 2.0.0-accumulo-2.0.1 # managed version
+ tag: 2.1.0-accumulo-2.0.1 # managed version
  pullPolicy: IfNotPresent
  pullSecrets: []
  resources: {}
@@ -61,7 +61,7 @@ accumulo:
  enabled: false
  image:
  repository: gchq/gaffer
- tag: 2.0.0-accumulo-2.0.1 # managed version
+ tag: 2.1.0-accumulo-2.0.1 # managed version
  config:
  postInstallCommands:
  - users