Integration tests intermittently fail on Spark 3.4+

[wajda]

java.lang.IllegalStateException: Promise already completed.
22:18:31       at scala.concurrent.Promise.complete(Promise.scala:53)
22:18:31       at scala.concurrent.Promise.complete$(Promise.scala:52)
22:18:31       at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:197)
22:18:31       at scala.concurrent.Promise.success(Promise.scala:86)
22:18:31       at scala.concurrent.Promise.success$(Promise.scala:86)
22:18:31       at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:197)
22:18:31       at za.co.absa.spline.test.fixture.spline.LineageCaptor$$anon$1.capture(LineageCaptor.scala:37)
22:18:31       at za.co.absa.spline.test.fixture.spline.LineageCapturingDispatcher.send(LineageCapturingDispatcher.scala:26)
22:18:31       at za.co.absa.spline.harvester.dispatcher.CompositeLineageDispatcher.$anonfun$send$1(CompositeLineageDispatcher.scala:39)
22:18:31       at za.co.absa.spline.harvester.dispatcher.CompositeLineageDispatcher.$anonfun$send$1$adapted(CompositeLineageDispatcher.scala:39)
22:18:31       at za.co.absa.spline.harvester.dispatcher.CompositeLineageDispatcher.$anonfun$withErrorHandling$1(CompositeLineageDispatcher.scala:51)
22:18:31       at za.co.absa.spline.harvester.dispatcher.CompositeLineageDispatcher.$anonfun$withErrorHandling$1$adapted(CompositeLineageDispatcher.scala:50)
22:18:31       at scala.collection.immutable.List.foreach(List.scala:431)
22:18:31       at za.co.absa.spline.harvester.dispatcher.CompositeLineageDispatcher.delegate(CompositeLineageDispatcher.scala:47)
22:18:31       at za.co.absa.spline.harvester.dispatcher.CompositeLineageDispatcher.send(CompositeLineageDispatcher.scala:39)
22:18:31       at za.co.absa.spline.agent.SplineAgent$$anon$1.$anonfun$handle$2(SplineAgent.scala:94)
22:18:31       at za.co.absa.spline.agent.SplineAgent$$anon$1.$anonfun$handle$2$adapted(SplineAgent.scala:92)
22:18:31       at scala.Option.foreach(Option.scala:407)
22:18:31       at za.co.absa.spline.agent.SplineAgent$$anon$1.$anonfun$handle$1(SplineAgent.scala:92)
22:18:31       at za.co.absa.spline.agent.SplineAgent$$anon$1.withErrorHandling(SplineAgent.scala:100)
22:18:31       at za.co.absa.spline.agent.SplineAgent$$anon$1.handle(SplineAgent.scala:72)
22:18:31       at za.co.absa.spline.harvester.listener.QueryExecutionListenerDelegate.onSuccess(QueryExecutionListenerDelegate.scala:28)
22:18:31       at org.apache.spark.sql.util.ExecutionListenerBus.doPostEvent(QueryExecutionListener.scala:173)
22:18:31       at org.apache.spark.sql.util.ExecutionListenerBus.doPostEvent(QueryExecutionListener.scala:143)
22:18:31       at org.apache.spark.util.ListenerBus.postToAll(ListenerBus.scala:117)
22:18:31       at org.apache.spark.util.ListenerBus.postToAll$(ListenerBus.scala:101)
22:18:31       at org.apache.spark.sql.util.ExecutionListenerBus.postToAll(QueryExecutionListener.scala:143)
22:18:31       at org.apache.spark.sql.util.ExecutionListenerBus.onOtherEvent(QueryExecutionListener.scala:155)
22:18:31       at org.apache.spark.scheduler.SparkListenerBus.doPostEvent(SparkListenerBus.scala:100)
22:18:31       at org.apache.spark.scheduler.SparkListenerBus.doPostEvent$(SparkListenerBus.scala:28)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)
22:18:31       at org.apache.spark.util.ListenerBus.postToAll(ListenerBus.scala:117)
22:18:31       at org.apache.spark.util.ListenerBus.postToAll$(ListenerBus.scala:101)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue.super$postToAll(AsyncEventQueue.scala:105)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue.$anonfun$dispatch$1(AsyncEventQueue.scala:105)
22:18:31       at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
22:18:31       at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:100)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.$anonfun$run$1(AsyncEventQueue.scala:96)
22:18:31       at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1356)
22:18:31       at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.run(AsyncEventQueue.scala:96)