Gh-252: Kerberos integration tests (#335)

* Add docker compose and config files for Integration tests with Kerberos

* Improve dockerfile apt install commands
This prevents getting stuck on installer messages which prompt for user input

docker/gaffer-kerberos/gaffer-integration-tests-krb/.env

@@ -0,0 +1,17 @@
+GAFFER_VERSION=2.0.0
+GAFFER_KRB_PASSWORD=change-password-for-gaffer
+GAFFER_TESTER_VERSION=2.0.0
+
+ZOOKEEPER_VERSION=3.7.1
+ZOOKEEPER_KRB_PASSWORD=change-password-for-zookeeper
+
+ACCUMULO_VERSION=1.9.3
+ACCUMULO_CONF_DIR=/etc/accumulo/conf
+ACCUMULO_KRB_PASSWORD=change-password-for-accumulo
+
+HADOOP_VERSION=3.2.2
+HADOOP_CONF_DIR=/etc/hadoop/conf
+HADOOP_KRB_PASSWORD=change-password-for-hadoop
+
+DEBUG=0
+GAFFER_DEBUG=0

docker/gaffer-kerberos/gaffer-integration-tests-krb/Dockerfile

@@ -0,0 +1,32 @@
+# Copyright 2021-2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG GAFFER_VERSION=develop
+ARG ACCUMULO_VERSION=1.9.3
+
+ARG BASE_IMAGE_NAME=gchq/gaffer-integration-tests
+ARG BASE_IMAGE_TAG=${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION}
+
+FROM ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -qq update && \
+ apt-get -qq install -y krb5-kdc && \
+ apt-get -qq clean && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY ./native /tmp/hadoop/native
+COPY ./krb-entrypoint.sh .
+ENTRYPOINT ["/bin/bash","./krb-entrypoint.sh"]

docker/gaffer-kerberos/gaffer-integration-tests-krb/README.md

@@ -0,0 +1,19 @@
+Gaffer Kerberos Integration Tests
+=================================
+This folder contains a Dockerfile for running integration tests against an Accumulo cluster which uses Kerberos authentication.
+
+For more information on the integration tests, please see the primary Gaffer Docker integration tests README.
+
+# Prerequisites
+For the HDFS tests to work, you must acquire and place the HDFS native libraries into the `native` directory.
+You must also have built the Gaffer with Kerberos containers in the directory above and the non-kerberos
+version of the integration tests container image. 
+
+# Running Locally
+These services can be built and run using docker compose:
+```bash
+docker compose up
+```
+
+# Issues
+HDFS tests fail with Accumulo 2.0.0. They pass with Accumulo 1.9.3. This problem has been raised as [Gaffer issue #3134](https://github.com/gchq/Gaffer/issues/3134).

docker/gaffer-kerberos/gaffer-integration-tests-krb/accumulo-int-entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# Copyright 2023 Crown Copyright
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wait for Accumulo to be started and working (uses same approach as compose healthcheck)
+until cat /proc/net/tcp | grep 270F; do
+ sleep 15
+done
+
+# Grant required permissions and auths to Gaffer user for integration tests
+
+PRINCIPAL="accumulo/$(hostname)"
+FULL_PRINCIPAL="$PRINCIPAL@GAFFER.DOCKER"
+GAFFER_FULL_PRINCIPAL=$1
+
+kinit -k -t /etc/accumulo/conf/accumulo.keytab $FULL_PRINCIPAL
+
+echo "\nGranting permissions for Gaffer integration tests\n"
+
+if echo "$ACCUMULO_VERSION" | grep -q "^2.*$"; then
+ ACCUMULO_SHELL_CMD="accumulo shell --config-file accumulo-shell-client.properties -e"
+else
+ ACCUMULO_SHELL_CMD="accumulo shell -e"
+fi
+
+$ACCUMULO_SHELL_CMD "createuser $GAFFER_FULL_PRINCIPAL"
+$ACCUMULO_SHELL_CMD "grant System.CREATE_TABLE -s -u $GAFFER_FULL_PRINCIPAL"
+$ACCUMULO_SHELL_CMD "grant System.DROP_TABLE -s -u $GAFFER_FULL_PRINCIPAL"
+$ACCUMULO_SHELL_CMD "grant System.ALTER_TABLE -s -u $GAFFER_FULL_PRINCIPAL"
+$ACCUMULO_SHELL_CMD "setauths -s vis1,vis2,publicVisibility,privateVisibility,public,private -u $GAFFER_FULL_PRINCIPAL"

docker/gaffer-kerberos/gaffer-integration-tests-krb/conf/accumulo-shell-client.properties

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Required to set Zookeeper host
+instance.zookeepers=zookeeper.gaffer:2181
+## Required to set Accumulo Instance name
+instance.name = accumulo
+## Required to get Kerberos to be used
+auth.type = kerberos
+auth.principal = accumulo/accumulo-master.gaffer@GAFFER.DOCKER
+auth.token = /etc/accumulo/conf/accumulo.keytab
+
+## Further Kerberos Config
+sasl.enabled = true
+sasl.qop = auth
+sasl.kerberos.server.primary = accumulo

docker/gaffer-kerberos/gaffer-integration-tests-krb/conf/core-site.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2022-2023 Crown Copyright
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://hdfs-namenode.gaffer:9000</value>
+ </property>
+</configuration>

docker/gaffer-kerberos/gaffer-integration-tests-krb/conf/hdfs-site.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2022-2023 Crown Copyright
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <!-- Local principal to use -->
+ <property><name>yarn.resourcemanager.principal</name><value>gaffer/gaffer-int.gaffer@GAFFER.DOCKER</value></property>
+ <!-- Principal of the datanode we are connecting to -->
+ <property><name>dfs.namenode.kerberos.principal</name><value>hadoop/hdfs-namenode.gaffer@GAFFER.DOCKER</value></property>
+</configuration>

docker/gaffer-kerberos/gaffer-integration-tests-krb/conf/log4j.xml

@@ -0,0 +1,33 @@
+<?xml version='1.0' encoding='UTF-8' ?>
+<!--
+ ~ Copyright 2017-2023 Crown Copyright
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
+ debug="false">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%c{3} %p %x - %m%n"/>
+ </layout>
+ </appender>
+ <root>
+ <priority value="info"></priority>
+ <appender-ref ref="console"></appender-ref>
+ </root>
+ <logger name="org.apache.hadoop.util.NativeCodeLoader">
+ <level value="DEBUG" />
+ </logger>
+</log4j:configuration>