From da0fc59fe2c40f8e39004efcbcfcab7359096eee Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:45:29 +0200 Subject: [PATCH 1/3] Add batch composite sink (#4105) * Initial batch BG sink * Remove custom blank nodes from construct-query.sparql * Adapt tests to search query changes * Simplify NewCompositeSink * Update query in CompositeIndexingSuite * Failed elems shouldn't be mapped to dropped. Chunks of dropped elems should still go to the sink * Clean imports * Update query in integration test * Resolve issues with BatchCompositeSink * Update CompositeIndexingSuite to work with new sink * Update the multi-construct-query.sparql to use an alias * Update queries in test resources * Add `drop` method on Elem * Rename `NewQueryGraph` to `BatchQueryGraph` * Fix neuronDensity and layerThickness in the mutli id case * Further "series" related fixes * Update sparql queries in integration tests * Group composite sinks * Change BatchQueryGraph signature * Make the choice of sink configurable * Remove "legacy" sparql query * Add default sink-config * Template `query` fields inside the composite-view payloads * The sink is not legacy * Reflect sinkConfig change in composite config * Refactor CompositeIndexingSuite so that it can run for both Single and Batch sink * Try out for comprehension for clarity * Add missing bracket * Remove unused QueryGraph trait * Split QueryGraph --- .../src/main/resources/composite-views.conf | 3 + .../compositeviews/CompositeSink.scala | 199 +++++++++++++++++- .../CompositeViewsPluginModule.scala | 2 +- .../config/CompositeViewsConfig.scala | 27 ++- .../indexing/BatchQueryGraph.scala | 47 +++++ .../indexing/CompositeSpaces.scala | 40 +--- ...ueryGraph.scala => SingleQueryGraph.scala} | 9 +- .../CompositeViewsFixture.scala | 5 +- .../indexing/CompositeIndexingSuite.scala | 188 +++++++++++------ .../nexus/delta/sourcing/stream/Elem.scala | 7 + tests/docker/config/construct-query.sparql | 131 ++++++------ tests/docker/config/delta-postgres.conf | 1 + .../composite-view-include-context.json | 4 +- .../kg/views/composite/composite-view.json | 4 +- .../nexus/tests/kg/CompositeViewsSpec.scala | 93 +++++++- .../nexus/tests/kg/SearchConfigSpec.scala | 2 + 16 files changed, 574 insertions(+), 188 deletions(-) create mode 100644 delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/BatchQueryGraph.scala rename delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/{QueryGraph.scala => SingleQueryGraph.scala} (85%) diff --git a/delta/plugins/composite-views/src/main/resources/composite-views.conf b/delta/plugins/composite-views/src/main/resources/composite-views.conf index 1a7e40a794..c691e14bd3 100644 --- a/delta/plugins/composite-views/src/main/resources/composite-views.conf +++ b/delta/plugins/composite-views/src/main/resources/composite-views.conf @@ -45,4 +45,7 @@ plugins.composite-views { # set to false to disable composite view indexing indexing-enabled = ${app.defaults.indexing.enable} + + # type of composite sink to use for composite view indexing + sink-config = single } \ No newline at end of file diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala index d4229a9acb..2175b4b02b 100644 --- a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala @@ -1,7 +1,23 @@ package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews import cats.implicits._ -import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.QueryGraph +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.indexing.{BlazegraphSink, GraphResourceToNTriples} +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.SinkConfig +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.SinkConfig.SinkConfig +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.{BatchQueryGraph, SingleQueryGraph} +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection.{ElasticSearchProjection, SparqlProjection} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient.Refresh +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearchClient, IndexLabel} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.{ElasticSearchSink, GraphResourceToDocument} +import ch.epfl.bluebrain.nexus.delta.rdf.graph.Graph +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri +import ch.epfl.bluebrain.nexus.delta.sourcing.config.BatchConfig import ch.epfl.bluebrain.nexus.delta.sourcing.state.GraphResource import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem.{DroppedElem, FailedElem, SuccessElem} @@ -12,6 +28,12 @@ import shapeless.Typeable import scala.concurrent.duration.FiniteDuration +/** + * A composite sink handles querying the common blazegraph namespace, transforming the result into a format that can be + * pushed into a target namespace or index, and finally sinks it into the target. + */ +trait CompositeSink extends Sink + /** * A sink that queries N-Triples in Blazegraph, transforms them, and pushes the result to the provided sink * @param queryGraph @@ -19,21 +41,21 @@ import scala.concurrent.duration.FiniteDuration * @param transform * function to transform a graph into the format needed by the sink * @param sink - * function that allows + * function that defines how to sink a chunk of Elem[SinkFormat] * @param chunkSize - * the maximum number of elements to be pushed in ES at once + * the maximum number of elements to be pushed into the sink * @param maxWindow - * the maximum number of elements to be pushed at once + * the maximum time to wait for the chunk to gather [[chunkSize]] elements * @tparam SinkFormat * the type of data accepted by the sink */ -final class CompositeSink[SinkFormat]( - queryGraph: QueryGraph, +final class Single[SinkFormat]( + queryGraph: SingleQueryGraph, transform: GraphResource => Task[Option[SinkFormat]], sink: Chunk[Elem[SinkFormat]] => Task[Chunk[Elem[Unit]]], override val chunkSize: Int, override val maxWindow: FiniteDuration -) extends Sink { +) extends CompositeSink { override type In = GraphResource override def inType: Typeable[GraphResource] = Typeable[GraphResource] @@ -54,3 +76,166 @@ final class CompositeSink[SinkFormat]( .flatMap(sink) } + +/** + * A sink that queries N-Triples in Blazegraph for multiple resources, transforms it for each resource, and pushes the + * result to the provided sink + * @param queryGraph + * how to query the blazegraph + * @param transform + * function to transform a graph into the format needed by the sink + * @param sink + * function that defines how to sink a chunk of Elem[SinkFormat] + * @param chunkSize + * the maximum number of elements to be pushed into the sink + * @param maxWindow + * the maximum time to wait for the chunk to gather [[chunkSize]] elements + * @tparam SinkFormat + * the type of data accepted by the sink + */ +final class Batch[SinkFormat]( + queryGraph: BatchQueryGraph, + transform: GraphResource => Task[Option[SinkFormat]], + sink: Chunk[Elem[SinkFormat]] => Task[Chunk[Elem[Unit]]], + override val chunkSize: Int, + override val maxWindow: FiniteDuration +)(implicit rcr: RemoteContextResolution) + extends CompositeSink { + + override type In = GraphResource + + override def inType: Typeable[GraphResource] = Typeable[GraphResource] + + /** Performs the sparql query only using [[SuccessElem]]s from the chunk */ + private def query(elements: Chunk[Elem[GraphResource]]): Task[Option[Graph]] = + elements.mapFilter(elem => elem.map(_.id).toOption) match { + case ids if ids.nonEmpty => queryGraph(ids) + case _ => Task.none + } + + /** Replaces the graph of a provided [[GraphResource]] by extracting its new graph from the provided (full) graph. */ + private def replaceGraph(gr: GraphResource, fullGraph: Graph) = { + implicit val api: JsonLdApi = JsonLdJavaApi.lenient + fullGraph + .replaceRootNode(iri"${gr.id}/alias") + .toCompactedJsonLd(ContextValue.empty) + .flatMap(_.toGraph) + .map(g => gr.copy(graph = g.replaceRootNode(gr.id))) + } + + override def apply(elements: Chunk[Elem[GraphResource]]): Task[Chunk[Elem[Unit]]] = + for { + graph <- query(elements) + transformed <- graph match { + case Some(fullGraph) => + elements.traverse { elem => + elem.evalMapFilter { gr => + replaceGraph(gr, fullGraph).flatMap(transform) + } + } + case None => + Task.pure(elements.map(_.drop)) + } + sank <- sink(transformed) + } yield sank +} + +object CompositeSink { + + /** + * @param blazeClient + * client used to connect to blazegraph + * @param namespace + * name of the target blazegraph namespace + * @param common + * name of the common blazegraph namespace + * @param cfg + * configuration of the composite views + * @return + * a function that given a sparql view returns a composite sink that has the view as target + */ + def blazeSink( + blazeClient: BlazegraphClient, + namespace: String, + common: String, + cfg: CompositeViewsConfig + )(implicit baseUri: BaseUri, rcr: RemoteContextResolution): SparqlProjection => CompositeSink = { target => + compositeSink( + blazeClient, + common, + target.query, + GraphResourceToNTriples.graphToNTriples, + BlazegraphSink(blazeClient, cfg.blazegraphBatch, namespace).apply, + cfg.blazegraphBatch, + cfg.sinkConfig + ) + } + + /** + * @param blazeClient + * blazegraph client used to query the common space + * @param esClient + * client used to push to elasticsearch + * @param index + * name of the target elasticsearch index + * @param common + * name of the common blazegraph namespace + * @param cfg + * configuration of the composite views + * @return + * a function that given a elasticsearch view returns a composite sink that has the view as target + */ + def elasticSink( + blazeClient: BlazegraphClient, + esClient: ElasticSearchClient, + index: IndexLabel, + common: String, + cfg: CompositeViewsConfig + )(implicit rcr: RemoteContextResolution): ElasticSearchProjection => CompositeSink = { target => + val esSink = + ElasticSearchSink.states( + esClient, + cfg.elasticsearchBatch.maxElements, + cfg.elasticsearchBatch.maxInterval, + index, + Refresh.False + ) + compositeSink( + blazeClient, + common, + target.query, + new GraphResourceToDocument(target.context, target.includeContext).graphToDocument, + esSink.apply, + cfg.elasticsearchBatch, + cfg.sinkConfig + ) + } + + private def compositeSink[SinkFormat]( + blazeClient: BlazegraphClient, + common: String, + query: SparqlConstructQuery, + transform: GraphResource => Task[Option[SinkFormat]], + sink: Chunk[Elem[SinkFormat]] => Task[Chunk[Elem[Unit]]], + batchConfig: BatchConfig, + sinkConfig: SinkConfig + )(implicit rcr: RemoteContextResolution): CompositeSink = sinkConfig match { + case SinkConfig.Single => + new Single( + new SingleQueryGraph(blazeClient, common, query), + transform, + sink, + batchConfig.maxElements, + batchConfig.maxInterval + ) + case SinkConfig.Batch => + new Batch( + new BatchQueryGraph(blazeClient, common, query), + transform, + sink, + batchConfig.maxElements, + batchConfig.maxInterval + ) + } + +} diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsPluginModule.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsPluginModule.scala index 1a1d76aaf4..6dfcf2f718 100644 --- a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsPluginModule.scala +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsPluginModule.scala @@ -164,7 +164,7 @@ class CompositeViewsPluginModule(priority: Int) extends ModuleDef { baseUri: BaseUri, cr: RemoteContextResolution @Id("aggregate") ) => - CompositeSpaces.Builder(cfg.prefix, esClient, cfg.elasticsearchBatch, blazeClient, cfg.blazegraphBatch)( + CompositeSpaces.Builder(cfg.prefix, esClient, blazeClient, cfg)( baseUri, cr ) diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/config/CompositeViewsConfig.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/config/CompositeViewsConfig.scala index 5310fc9918..5e89ea192a 100644 --- a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/config/CompositeViewsConfig.scala +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/config/CompositeViewsConfig.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config import akka.http.scaladsl.model.Uri import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.config.BlazegraphViewsConfig.Credentials +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.SinkConfig.SinkConfig import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.{BlazegraphAccess, RemoteSourceClientConfig, SourcesConfig} import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClientConfig import ch.epfl.bluebrain.nexus.delta.sdk.instances._ @@ -9,6 +10,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.model.search.PaginationConfig import ch.epfl.bluebrain.nexus.delta.sourcing.config.{BatchConfig, EventLogConfig} import com.typesafe.config.Config import monix.bio.UIO +import pureconfig.error.CannotConvert import pureconfig.generic.auto._ import pureconfig.generic.semiauto.deriveReader import pureconfig.{ConfigReader, ConfigSource} @@ -42,6 +44,8 @@ import scala.concurrent.duration.{Duration, FiniteDuration} * the interval at which a view will look for requested restarts * @param indexingEnabled * if false, disables composite view indexing + * @param sinkConfig + * type of sink used for composite indexing */ final case class CompositeViewsConfig( sources: SourcesConfig, @@ -55,7 +59,8 @@ final case class CompositeViewsConfig( blazegraphBatch: BatchConfig, elasticsearchBatch: BatchConfig, restartCheckInterval: FiniteDuration, - indexingEnabled: Boolean + indexingEnabled: Boolean, + sinkConfig: SinkConfig ) object CompositeViewsConfig { @@ -106,6 +111,26 @@ object CompositeViewsConfig { maxTimeWindow: FiniteDuration ) + object SinkConfig { + + /** Represents the choice of composite sink */ + sealed trait SinkConfig + + /** A sink that only supports querying one resource at once from blazegraph */ + case object Single extends SinkConfig + + /** A sink that supports querying multiple resources at once from blazegraph */ + case object Batch extends SinkConfig + + implicit val sinkConfigReaderString: ConfigReader[SinkConfig] = + ConfigReader.fromString { + case "batch" => Right(Batch) + case "single" => Right(Single) + case value => + Left(CannotConvert(value, SinkConfig.getClass.getSimpleName, s"$value is not one of: [single, batch]")) + } + } + /** * Converts a [[Config]] into an [[CompositeViewsConfig]] */ diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/BatchQueryGraph.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/BatchQueryGraph.scala new file mode 100644 index 0000000000..30f253af29 --- /dev/null +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/BatchQueryGraph.scala @@ -0,0 +1,47 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing + +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.SparqlQueryResponseType.SparqlNTriples +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection.idTemplating +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.graph.{Graph, NTriples} +import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery +import fs2.Chunk +import monix.bio.Task + +import java.util.regex.Pattern.quote + +/** + * Provides a way to query for the multiple incoming resources (from a chunk). This assumes that the query contains the + * template: `VALUE ?id { {resource_id} }`. The result is a single Graph for all given resources. + * @param client + * the blazegraph client used to query + * @param namespace + * the namespace to query + * @param query + * the sparql query to perform + */ +final class BatchQueryGraph(client: BlazegraphClient, namespace: String, query: SparqlConstructQuery) { + + private val logger: Logger = Logger[BatchQueryGraph] + + private def newGraph(ntriples: NTriples): Task[Option[Graph]] = + if (ntriples.isEmpty) Task.none + else Task.fromEither(Graph(ntriples)).map(Some(_)) + + def apply(ids: Chunk[Iri]): Task[Option[Graph]] = + for { + ntriples <- client.query(Set(namespace), replaceIds(query, ids), SparqlNTriples) + graphResult <- newGraph(ntriples.value) + _ <- Task.when(graphResult.isEmpty)( + logger.debug(s"Querying blazegraph did not return any triples, '$ids' will be dropped.") + ) + } yield graphResult + + private def replaceIds(query: SparqlConstructQuery, iris: Chunk[Iri]): SparqlConstructQuery = { + val replacement = iris.foldLeft("") { (acc, iri) => acc + " " + s"<$iri>" } + SparqlConstructQuery.unsafe(query.value.replaceAll(quote(idTemplating), replacement)) + } + +} diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeSpaces.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeSpaces.scala index b4399ea2be..d073a86f4d 100644 --- a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeSpaces.scala +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeSpaces.scala @@ -2,18 +2,16 @@ package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient -import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.indexing.{BlazegraphSink, GraphResourceToNTriples} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.indexing.BlazegraphSink import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeSink +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.CompositeViewDef.ActiveViewDef import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection.{ElasticSearchProjection, SparqlProjection} -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient.Refresh import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearchClient, IndexLabel} -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.{ElasticSearchSink, GraphResourceToDocument} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri -import ch.epfl.bluebrain.nexus.delta.sourcing.config.BatchConfig import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Operation.Sink import com.typesafe.scalalogging.Logger import monix.bio.Task @@ -54,41 +52,21 @@ object CompositeSpaces { def apply( prefix: String, esClient: ElasticSearchClient, - esBatchConfig: BatchConfig, blazeClient: BlazegraphClient, - blazeBatchConfig: BatchConfig + cfg: CompositeViewsConfig )(implicit base: BaseUri, rcr: RemoteContextResolution): CompositeSpaces.Builder = (view: ActiveViewDef) => { // Operations and sinks for the common space val common = commonNamespace(view.uuid, view.rev, prefix) - val commonSink = BlazegraphSink(blazeClient, blazeBatchConfig, common) + val commonSink = BlazegraphSink(blazeClient, cfg.blazegraphBatch, common) val createCommon = blazeClient.createNamespace(common) val deleteCommon = blazeClient.deleteNamespace(common) - // Create the Blazegraph sink - def createBlazeSink(namespace: String): SparqlProjection => Sink = { target => - val blazeSink = BlazegraphSink(blazeClient, blazeBatchConfig, namespace) - new CompositeSink( - QueryGraph(blazeClient, common, target.query), - GraphResourceToNTriples.graphToNTriples, - blazeSink.apply, - blazeBatchConfig.maxElements, - blazeBatchConfig.maxInterval - ) - } - - // Create the Elasticsearch index - def createEsSink(index: IndexLabel): ElasticSearchProjection => Sink = { target => - val esSink = - ElasticSearchSink.states(esClient, esBatchConfig.maxElements, esBatchConfig.maxInterval, index, Refresh.False) - new CompositeSink( - QueryGraph(blazeClient, common, target.query), - new GraphResourceToDocument(target.context, target.includeContext).graphToDocument, - esSink.apply, - esBatchConfig.maxElements, - esBatchConfig.maxInterval - ) - } + // Create sinks + def createBlazeSink(namespace: String): SparqlProjection => Sink = + CompositeSink.blazeSink(blazeClient, namespace, common, cfg) + def createEsSink(index: IndexLabel): ElasticSearchProjection => Sink = + CompositeSink.elasticSink(blazeClient, esClient, index, common, cfg) // Compute the init and destroy operations as well as the sink for the different projections of the composite views val start: (Task[Unit], Task[Unit], Map[Iri, Sink]) = (createCommon.void, deleteCommon.void, Map.empty[Iri, Sink]) diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/QueryGraph.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/SingleQueryGraph.scala similarity index 85% rename from delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/QueryGraph.scala rename to delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/SingleQueryGraph.scala index 6ccc375134..65a4b3fda3 100644 --- a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/QueryGraph.scala +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/SingleQueryGraph.scala @@ -13,17 +13,18 @@ import monix.bio.Task import java.util.regex.Pattern.quote /** - * Pipe that performs the provided query for the incoming resource and replaces the graph with the result of query + * Provides a way to query for the incoming resource and replaces the graph with the result of query + * * @param client - * the blazegraph client + * the blazegraph client used to query * @param namespace * the namespace to query * @param query * the query to perform on each resource */ -final case class QueryGraph(client: BlazegraphClient, namespace: String, query: SparqlConstructQuery) { +final class SingleQueryGraph(client: BlazegraphClient, namespace: String, query: SparqlConstructQuery) { - private val logger: Logger = Logger[QueryGraph] + private val logger: Logger = Logger[SingleQueryGraph] private def newGraph(ntriples: NTriples, id: Iri): Task[Option[Graph]] = if (ntriples.isEmpty) { diff --git a/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsFixture.scala b/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsFixture.scala index e5f872f7cf..1efa8ab7f8 100644 --- a/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsFixture.scala +++ b/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeViewsFixture.scala @@ -5,7 +5,7 @@ import cats.data.NonEmptySet import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig -import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.{BlazegraphAccess, RemoteSourceClientConfig, SourcesConfig} +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.{BlazegraphAccess, RemoteSourceClientConfig, SinkConfig, SourcesConfig} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeView.Interval import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection.{ElasticSearchProjection, SparqlProjection} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjectionFields.{ElasticSearchProjectionFields, SparqlProjectionFields} @@ -178,7 +178,8 @@ trait CompositeViewsFixture extends ConfigFixtures with EitherValuable { batchConfig, batchConfig, 3.seconds, - false + false, + SinkConfig.Batch ) } diff --git a/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeIndexingSuite.scala b/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeIndexingSuite.scala index 721e14bc7f..a6fc5c3a75 100644 --- a/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeIndexingSuite.scala +++ b/delta/plugins/composite-views/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/indexing/CompositeIndexingSuite.scala @@ -2,16 +2,20 @@ package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing import akka.http.scaladsl.model.Uri import akka.http.scaladsl.model.Uri.Query +import cats.Semigroup import cats.data.{NonEmptyList, NonEmptySet} import cats.effect.Resource import cats.effect.concurrent.Ref -import cats.kernel.Semigroup import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.search.Pagination.FromPagination import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.BlazegraphClientSetup import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.SparqlQueryResponseType.SparqlNTriples -import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.CompositeIndexingSuite.{batchConfig, Album, Band, Music} +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.SinkConfig +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.config.CompositeViewsConfig.SinkConfig.SinkConfig +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.CompositeIndexingSuite._ import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.CompositeViewDef.ActiveViewDef +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.Queries.{batchQuery, singleQuery} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection.{ElasticSearchProjection, SparqlProjection} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewSource.{CrossProjectSource, ProjectSource, RemoteProjectSource} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{permissions, CompositeView, CompositeViewSource, CompositeViewValue} @@ -19,20 +23,20 @@ import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.projections.Composit import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.store.CompositeRestartStore import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.stream.CompositeBranch.Run.{Main, Rebuild} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.stream.{CompositeBranch, CompositeGraphStream, CompositeProgress} -import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.{CompositeViews, Fixtures} +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.{CompositeViews, CompositeViewsFixture, Fixtures} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.ElasticSearchClientSetup import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearchClient, IndexLabel, QueryBuilder} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{nxv, rdf, rdfs, schemas} import ch.epfl.bluebrain.nexus.delta.rdf.graph.Graph -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue.ContextObject import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.{iriStringContextSyntax, jsonOpsSyntax} import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri -import ch.epfl.bluebrain.nexus.delta.kernel.search.Pagination.FromPagination import ch.epfl.bluebrain.nexus.delta.sdk.model.search.{Sort, SortList} import ch.epfl.bluebrain.nexus.delta.sdk.views.ViewRef import ch.epfl.bluebrain.nexus.delta.sourcing.config.{BatchConfig, QueryConfig} @@ -40,15 +44,15 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ElemStream, EntityType, Label, ProjectRef} import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset +import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie import ch.epfl.bluebrain.nexus.delta.sourcing.query.RefreshStrategy import ch.epfl.bluebrain.nexus.delta.sourcing.state.GraphResource import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem.SuccessElem import ch.epfl.bluebrain.nexus.delta.sourcing.stream._ import ch.epfl.bluebrain.nexus.delta.sourcing.stream.pipes.{DiscardMetadata, FilterByType, FilterDeprecated} +import ch.epfl.bluebrain.nexus.testkit.TestHelpers import ch.epfl.bluebrain.nexus.testkit.bio.ResourceFixture.TaskFixture -import ch.epfl.bluebrain.nexus.testkit.bio.{BioSuite, JsonAssertions, PatienceConfig, ResourceFixture, TextAssertions} -import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie -import ch.epfl.bluebrain.nexus.testkit.{IOFixedClock, TestHelpers} +import ch.epfl.bluebrain.nexus.testkit.bio._ import fs2.Stream import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.deriveConfiguredEncoder @@ -56,27 +60,68 @@ import io.circe.generic.semiauto.deriveEncoder import io.circe.syntax._ import io.circe.{Encoder, Json} import monix.bio.{Task, UIO} -import monix.execution.Scheduler import munit.AnyFixture import java.time.Instant import java.util.UUID -import scala.concurrent.duration._ +import scala.concurrent.duration.DurationInt + +class SingleCompositeIndexingSuite extends CompositeIndexingSuite(SinkConfig.Single, singleQuery) +class BatchCompositeIndexingSuite extends CompositeIndexingSuite(SinkConfig.Batch, batchQuery) + +trait CompositeIndexingFixture extends BioSuite { + + implicit private val baseUri: BaseUri = BaseUri("http://localhost", Label.unsafe("v1")) + implicit private val rcr: RemoteContextResolution = RemoteContextResolution.never + + private val queryConfig = QueryConfig(10, RefreshStrategy.Delay(10.millis)) + val batchConfig = BatchConfig(2, 50.millis) + private val compositeConfig = + CompositeViewsFixture.config.copy( + blazegraphBatch = batchConfig, + elasticsearchBatch = batchConfig + ) + + type Result = (ElasticSearchClient, BlazegraphClient, CompositeProjections, CompositeSpaces.Builder) + + private def resource(sinkConfig: SinkConfig): Resource[Task, Result] = { + (Doobie.resource(), ElasticSearchClientSetup.resource(), BlazegraphClientSetup.resource()).parMapN { + case (xas, esClient, bgClient) => + val compositeRestartStore = new CompositeRestartStore(xas) + val projections = + CompositeProjections(compositeRestartStore, xas, queryConfig, batchConfig, 3.seconds) + val spacesBuilder = + CompositeSpaces.Builder("delta", esClient, bgClient, compositeConfig.copy(sinkConfig = sinkConfig))( + baseUri, + rcr + ) + (esClient, bgClient, projections, spacesBuilder) + } + } + + def suiteLocalFixture(name: String, sinkConfig: SinkConfig): TaskFixture[Result] = + ResourceFixture.suiteLocal(name, resource(sinkConfig)) + + def compositeIndexing(sinkConfig: SinkConfig): ResourceFixture.TaskFixture[Result] = + suiteLocalFixture("compositeIndexing", sinkConfig) + +} -class CompositeIndexingSuite - extends BioSuite - with CompositeIndexingSuite.Fixture +abstract class CompositeIndexingSuite(sinkConfig: SinkConfig, query: SparqlConstructQuery) + extends CompositeIndexingFixture with TestHelpers with Fixtures with JsonAssertions with TextAssertions { - override def munitFixtures: Seq[AnyFixture[_]] = List(compositeIndexing) + private val fixture = compositeIndexing(sinkConfig) + + override def munitFixtures: Seq[AnyFixture[_]] = List(fixture) implicit private val patienceConfig: PatienceConfig = PatienceConfig(10.seconds, 100.millis) private val prefix = "delta" - private lazy val (esClient, bgClient, projections, spacesBuilder) = compositeIndexing() + private lazy val (esClient, bgClient, projections, spacesBuilder) = fixture() // Data to index private val museId = iri"http://music.com/muse" @@ -95,27 +140,6 @@ class CompositeIndexingSuite private val theGatewayId = iri"http://music.com/the_getaway" private val theGateway = Album(theGatewayId, "The Getaway", redHotId) - private val query = SparqlConstructQuery.unsafe( - """ - |prefix music: - |CONSTRUCT { - | {resource_id} music:name ?bandName ; - | music:genre ?bandGenre ; - | music:start ?bandStartYear ; - | music:album ?albumId . - | ?albumId music:title ?albumTitle . - |} WHERE { - | {resource_id} music:name ?bandName ; - | music:start ?bandStartYear; - | music:genre ?bandGenre . - | OPTIONAL { - | {resource_id} ^music:by ?albumId . - | ?albumId music:title ?albumTitle . - | } - |} - |""".stripMargin - ) - private val project1 = ProjectRef.unsafe("org", "proj") private val project2 = ProjectRef.unsafe("org", "proj2") private val project3 = ProjectRef.unsafe("org", "proj3") @@ -180,7 +204,8 @@ class CompositeIndexingSuite private val mainCompleted = Ref.unsafe[Task, Map[ProjectRef, Int]](Map.empty) private val rebuildCompleted = Ref.unsafe[Task, Map[ProjectRef, Int]](Map.empty) - private def resetCompleted = mainCompleted.set(Map.empty) >> rebuildCompleted.set(Map.empty) + private def resetCompleted = mainCompleted.set(Map.empty) >> rebuildCompleted.set(Map.empty) + private def increment(map: Ref[Task, Map[ProjectRef, Int]], project: ProjectRef) = map.update(_.updatedWith(project)(_.map(_ + 1).orElse(Some(1)))) @@ -529,41 +554,17 @@ class CompositeIndexingSuite } yield () } - private val checkQuery = SparqlConstructQuery.unsafe("CONSTRUCT {?s ?p ?o} WHERE {?s ?p ?o} ORDER BY ?s") + private val checkQuery = SparqlConstructQuery.unsafe("CONSTRUCT {?s ?p ?o} WHERE {?s ?p ?o} ORDER BY ?s") + private def checkBlazegraphTriples(namespace: String, expected: String) = bgClient .query(Set(namespace), checkQuery, SparqlNTriples) .map(_.value.toString) .map(_.equalLinesUnordered(expected)) -} - -object CompositeIndexingSuite extends IOFixedClock { - - implicit private val baseUri: BaseUri = BaseUri("http://localhost", Label.unsafe("v1")) - implicit private val rcr: RemoteContextResolution = RemoteContextResolution.never - - private val queryConfig: QueryConfig = QueryConfig(10, RefreshStrategy.Delay(10.millis)) - val batchConfig: BatchConfig = BatchConfig(2, 50.millis) - type Result = (ElasticSearchClient, BlazegraphClient, CompositeProjections, CompositeSpaces.Builder) - - private def resource()(implicit s: Scheduler, cl: ClassLoader): Resource[Task, Result] = { - (Doobie.resource(), ElasticSearchClientSetup.resource(), BlazegraphClientSetup.resource()).parMapN { - case (xas, esClient, bgClient) => - val compositeRestartStore = new CompositeRestartStore(xas) - val projections = - CompositeProjections(compositeRestartStore, xas, queryConfig, batchConfig, 3.seconds) - val spacesBuilder = CompositeSpaces.Builder("delta", esClient, batchConfig, bgClient, batchConfig)(baseUri, rcr) - (esClient, bgClient, projections, spacesBuilder) - } - } - - def suiteLocalFixture(name: String)(implicit s: Scheduler, cl: ClassLoader): TaskFixture[Result] = - ResourceFixture.suiteLocal(name, resource()) +} - trait Fixture { self: BioSuite => - val compositeIndexing: ResourceFixture.TaskFixture[Result] = suiteLocalFixture("compositeIndexing") - } +object CompositeIndexingSuite { private val ctxIri = ContextValue(iri"http://music.com/context") @@ -576,7 +577,9 @@ object CompositeIndexingSuite extends IOFixedClock { sealed trait Music extends Product with Serializable { def id: Iri + def tpe: Iri + def label: String } @@ -584,16 +587,19 @@ object CompositeIndexingSuite extends IOFixedClock { override val tpe: Iri = iri"http://music.com/Band" override val label: String = name } + object Band { implicit val bandEncoder: Encoder.AsObject[Band] = deriveConfiguredEncoder[Band].mapJsonObject(_.add("@type", "Band".asJson)) implicit val bandJsonLdEncoder: JsonLdEncoder[Band] = JsonLdEncoder.computeFromCirce((b: Band) => b.id, ctxIri) } - final case class Album(id: Iri, title: String, by: Iri) extends Music { + + final case class Album(id: Iri, title: String, by: Iri) extends Music { override val tpe: Iri = iri"http://music.com/Album" override val label: String = title } + object Album { implicit val albumEncoder: Encoder.AsObject[Album] = deriveConfiguredEncoder[Album].mapJsonObject(_.add("@type", "Album".asJson)) @@ -602,6 +608,7 @@ object CompositeIndexingSuite extends IOFixedClock { } final case class Metadata(uuid: UUID) + object Metadata { implicit private val encoderMetadata: Encoder.AsObject[Metadata] = deriveEncoder implicit val jsonLdEncoderMetadata: JsonLdEncoder[Metadata] = JsonLdEncoder.computeFromCirce(ctxIri) @@ -609,3 +616,52 @@ object CompositeIndexingSuite extends IOFixedClock { } } + +object Queries { + val batchQuery: SparqlConstructQuery = SparqlConstructQuery.unsafe( + """ + |prefix music: + |CONSTRUCT { + | ?alias music:name ?bandName ; + | music:genre ?bandGenre ; + | music:start ?bandStartYear ; + | music:album ?albumId . + | ?albumId music:title ?albumTitle . + |} WHERE { + | VALUES ?id { {resource_id} } . + | BIND( IRI(concat(str(?id), '/', 'alias')) AS ?alias ) . + | + | ?id music:name ?bandName ; + | music:start ?bandStartYear; + | music:genre ?bandGenre . + | OPTIONAL { + | ?id ^music:by ?albumId . + | ?albumId music:title ?albumTitle . + | } + |} + |""".stripMargin + ) + + val singleQuery: SparqlConstructQuery = SparqlConstructQuery.unsafe( + """ + |prefix music: + |CONSTRUCT { + | ?id music:name ?bandName ; + | music:genre ?bandGenre ; + | music:start ?bandStartYear ; + | music:album ?albumId . + | ?albumId music:title ?albumTitle . + |} WHERE { + | BIND( {resource_id} AS ?id ) . + | + | ?id music:name ?bandName ; + | music:start ?bandStartYear; + | music:genre ?bandGenre . + | OPTIONAL { + | ?id ^music:by ?albumId . + | ?albumId music:title ?albumTitle . + | } + |} + |""".stripMargin + ) +} diff --git a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/stream/Elem.scala b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/stream/Elem.scala index 56060ec34e..63d05219e1 100644 --- a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/stream/Elem.scala +++ b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/stream/Elem.scala @@ -78,6 +78,13 @@ sealed trait Elem[+A] extends Product with Serializable { */ def dropped: DroppedElem = DroppedElem(tpe, id, project, instant, offset, rev) + /** Action of dropping an Elem */ + def drop: Elem[Nothing] = this match { + case e: SuccessElem[A] => e.dropped + case e: FailedElem => e + case e: DroppedElem => e + } + /** * Maps the underlying element value if this is a [[Elem.SuccessElem]] using f. * @param f diff --git a/tests/docker/config/construct-query.sparql b/tests/docker/config/construct-query.sparql index 77ba1b1b34..f7069580e9 100644 --- a/tests/docker/config/construct-query.sparql +++ b/tests/docker/config/construct-query.sparql @@ -10,7 +10,7 @@ prefix : CONSTRUCT { # Metadata - ?id a ?type ; + ?alias a ?type ; :name ?name ; :description ?description ; :createdAt ?createdAt ; @@ -20,68 +20,68 @@ CONSTRUCT { :deprecated ?deprecated ; :self ?self . - ?id :project ?projectBN . - ?projectBN :identifier ?projectId ; + ?alias :project ?projectId . + ?projectId :identifier ?projectId ; :label ?projectLabel . # Common properties ## Annotations - ?id :mType ?mType . + ?alias :mType ?mType . ?mType :identifier ?mType ; :label ?mTypeLabel ; :idLabel ?mTypeIdLabel . - ?id :eType ?eType . + ?alias :eType ?eType . ?eType :identifier ?eType ; :label ?eTypeLabel ; :idLabel ?eTypeIdLabel . - ?id :sType ?sType . + ?alias :sType ?sType . ?sType :identifier ?sType ; :label ?sTypeLabel ; :idLabel ?sTypeIdLabel . ## Atlas - ?id :coordinatesInBrainAtlas ?coordinates . + ?alias :coordinatesInBrainAtlas ?coordinates . ?coordinates :valueX ?valueX ; :valueY ?valueY ; :valueZ ?valueZ . ## Brain region / layer - ?id :brainRegion ?brainRegionId . - ?brainRegionId :identifier ?brainRegionId ; - :label ?brainRegionLabel ; - :idLabel ?brainRegionIdLabel . - ?id :layer ?layerId . + ?alias :brainRegion ?brainRegionId . + ?brainRegionId :identifier ?brainRegionId ; + :label ?brainRegionLabel ; + :idLabel ?brainRegionIdLabel . + ?alias :layer ?layerId . ?layerId :identifier ?layerId ; :label ?layerLabel ; :idLabel ?layerIdLabel . ## Contributors / Organizations - ?id :contributors ?personId . + ?alias :contributors ?personId . ?personId :identifier ?personId ; :label ?personName ; :idLabel ?personIdName ; :affiliation ?affiliation . ?affiliation :label ?affiliationName . - ?id :organizations ?orgId . + ?alias :organizations ?orgId . ?orgId :identifier ?orgId ; :label ?organizationName ; :idLabel ?organizationIdName . ## Derivation - ?id :derivation ?derivation . + ?alias :derivation ?derivation . ?derivation :identifier ?entity ; rdf:type ?entityType ; :label ?entityName . ## Distribution - ?id :distribution ?distribution . + ?alias :distribution ?distribution . ?distribution :label ?distributionName ; :encodingFormat ?distributionEncodingFormat ; :contentUrl ?distributionContentUrl ; :contentSize ?distributionContentSize . ## Generation - ?id :generation ?generation . + ?alias :generation ?generation . ?generation :protocol ?protocol ; :startedAt ?generationStartedAtTime ; :endedAt ?generationEndedAtTime . @@ -90,25 +90,25 @@ CONSTRUCT { :value ?protocolValue . ## Images - ?id :image ?image . + ?alias :image ?image . ?image :identifier ?image ; :about ?imageAbout ; :repetition ?imageRepetition ; :stimulusType ?imageStimulusTypeLabel . ## License - ?id :license ?licenseBN . - ?licenseBN :identifier ?licenseId ; + ?alias :license ?licenseId . + ?licenseId :identifier ?licenseId ; :label ?licenseLabel . ## Series - ?id :series ?series . + ?alias :series ?series . ?series :value ?seriesValue ; :unit ?seriesUnit ; :statistic ?seriesStatistic . ## Source - ?id :source ?source . + ?alias :source ?source . ?source :title ?sourceTitle ; rdf:type ?sourceType ; :identifier ?sourceIdentifier . @@ -116,79 +116,80 @@ CONSTRUCT { :value ?sourceIdentifierValue . ## Species - ?id :subjectSpecies ?species . + ?alias :subjectSpecies ?species . ?species :identifier ?species ; :label ?speciesLabel . ## Start / end / status - ?id :startedAt ?startedAt ; + ?alias :startedAt ?startedAt ; :endedAt ?endedAt ; :status ?status . ## Subject - ?id :subjectAge ?age . - ?age :value ?subjectAgeValue ; - :minValue ?subjectAgeMinValue ; - :maxValue ?subjectAgeMaxValue ; - :unit ?subjectAgeUnit ; - :period ?subjectAgePeriod ; - :label ?subjectAgeLabel . - ?id :subjectWeight ?subjectWeightBN . - ?subjectWeightBN :value ?subjectWeightValue ; - :unit ?subjectWeightUnit ; - :minValue ?subjectWeightMinValue ; - :maxValue ?subjectWeightMaxValue ; - :label ?subjectWeightLabel . + ?alias :subjectAge ?age . + ?age :value ?subjectAgeValue ; + :minValue ?subjectAgeMinValue ; + :maxValue ?subjectAgeMaxValue ; + :unit ?subjectAgeUnit ; + :period ?subjectAgePeriod ; + :label ?subjectAgeLabel . + ?alias :subjectWeight ?weight . + ?weight :value ?subjectWeightValue ; + :unit ?subjectWeightUnit ; + :minValue ?subjectWeightMinValue ; + :maxValue ?subjectWeightMaxValue ; + :label ?subjectWeightLabel . # Properties specific to types ## Bouton density - ?id :boutonDensity ?boutonDensityBN . - ?boutonDensityBN :value ?boutonDensityValue ; + ?alias :boutonDensity ?boutonDensityBN . + ?boutonDensityBN :value ?boutonDensityValue ; :unit ?boutonDensityUnit ; :label ?boutonDensityLabel . ## Circuit - ?id :circuitType ?circuitType ; + ?alias :circuitType ?circuitType ; :circuitBase ?circuitBaseUrlStr ; :circuitConfigPath ?circuitConfigPathUrlStr ; - :circuitBrainRegion ?circuitBrainRegionBN . - ?circuitBrainRegionBN :identifier ?circuitBrainRegionId ; + :circuitBrainRegion ?circuitBrainRegionId . + ?circuitBrainRegionId :identifier ?circuitBrainRegionId ; :label ?circuitBrainRegionLabel . ## Layer thickness - ?id :layerThickness ?thicknessBN . - ?thicknessBN :value ?thicknessValue ; + ?alias :layerThickness ?thicknessBN . + ?thicknessBN :value ?thicknessValue ; :unit ?thicknessUnit ; :nValue ?thicknessNValue ; :label ?thicknessLabel . ## Neuron density - ?id :neuronDensity ?neuronDensityBN . - ?neuronDensityBN :value ?neuronDensityValue ; + ?alias :neuronDensity ?neuronDensityBN . + ?neuronDensityBN :value ?neuronDensityValue ; :unit ?neuronDensityUnit ; :nValue ?neuronDensityNValue ; :label ?neuronDensityLabel . ## Simulation campaigns - ?id :config ?campaignConfigId . + ?alias :config ?campaignConfigId . ?campaignConfigId :identifier ?campaignConfigId ; :name ?campaignConfigName . ## Simulation campaigns / simulations - ?id :parameter ?parameter . + ?alias :parameter ?parameter . ?parameter :attrs ?attrs . ?attrs ?attrs_prop ?attrs_value . - ?id :parameter ?parameter . + ?alias :parameter ?parameter . ?parameter :coords ?coords . ?coords ?coords_prop ?coords_value . ## Simulations - ?id :campaign ?campaignId . + ?alias :campaign ?campaignId . ?campaignId :identifier ?campaignId ; :name ?campaignName . } WHERE { - BIND({resource_id} as ?id) . + VALUES ?id { {resource_id} } . + BIND( IRI(concat(str(?id), '/', 'alias')) AS ?alias ) . ?id a ?type . @@ -211,7 +212,6 @@ CONSTRUCT { nxv:self ?self ; nxv:project ?projectId . BIND( STRAFTER(STR(?projectId), "/v1/projects/") as ?projectLabel ) . - BIND( BNODE() AS ?projectBN ) . # We read the following nodes as text. This is done in order to avoid conflict # when another triple uses the same @id. For instance, createdBy and updatedBy @@ -401,10 +401,7 @@ CONSTRUCT { # License OPTIONAL { ?id schema:license ?licenseId . - OPTIONAL { - ?licenseId schema:name ?licenseLabel . - } . - BIND( BNODE() AS ?licenseBN ) . + OPTIONAL { ?licenseId schema:name ?licenseLabel . } . } . # Series @@ -469,7 +466,6 @@ CONSTRUCT { CONCAT(STR(?subjectWeightMinValue), " to ", STR(?subjectWeightMaxValue), " ", STR(?subjectWeightUnit)) ) as ?subjectWeightLabel ) . - BIND( BNODE() as ?subjectWeightBN ) . } . # Bouton density @@ -485,7 +481,7 @@ CONSTRUCT { STR(?boutonDensityUnit) ) as ?boutonDensityLabel ) . - BIND( BNODE() as ?boutonDensityBN ) . + BIND( BNODE(CONCAT(STR(?id), '/boutonDensity')) as ?boutonDensityBN ) . } . # Circuit @@ -524,12 +520,12 @@ CONSTRUCT { OPTIONAL { ?id a nsg:LayerThickness ; nsg:series ?meanSeries . - ?meanSeries nsg:statistic "mean" ; - schema:value ?thicknessValue ; - schema:unitCode ?thicknessUnit . - ?id nsg:series ?nSeries . - ?nSeries nsg:statistic "N" ; - schema:value ?thicknessNValue . + ?meanSeries nsg:statistic "mean" ; + schema:value ?thicknessValue ; + schema:unitCode ?thicknessUnit . + ?id nsg:series ?nSeries . + ?nSeries nsg:statistic "N" ; + schema:value ?thicknessNValue . BIND( CONCAT( STR(?thicknessValue), " ", @@ -537,7 +533,8 @@ CONSTRUCT { STR(?thicknessNValue), ")" ) as ?thicknessLabel ) . - BIND( BNODE() as ?thicknessBN ) . + + BIND( BNODE(CONCAT(STR(?id), '/layerThickness')) as ?thicknessBN ) . } . # Neuron density @@ -558,7 +555,8 @@ CONSTRUCT { STR(?neuronDensityNValue), ")" ) as ?neuronDensityLabel ) . - BIND( BNODE() as ?neuronDensityBN ) . + + BIND( BNODE((CONCAT(STR(?id), '/neuronDensity'))) as ?neuronDensityBN ) . } . # Simulation campaign configuration @@ -585,7 +583,6 @@ CONSTRUCT { OPTIONAL { ?circuitBrainRegionId rdfs:label ?circuitBrainRegionLabel . } - BIND( BNODE() AS ?circuitBrainRegionBN ) . } . } . } . diff --git a/tests/docker/config/delta-postgres.conf b/tests/docker/config/delta-postgres.conf index b27dded274..e5fe26cecd 100644 --- a/tests/docker/config/delta-postgres.conf +++ b/tests/docker/config/delta-postgres.conf @@ -55,6 +55,7 @@ plugins { composite-views { min-interval-rebuild = 5 seconds + sink-config = batch } elasticsearch { diff --git a/tests/src/test/resources/kg/views/composite/composite-view-include-context.json b/tests/src/test/resources/kg/views/composite/composite-view-include-context.json index 6afe1aeafe..6181f5546f 100644 --- a/tests/src/test/resources/kg/views/composite/composite-view-include-context.json +++ b/tests/src/test/resources/kg/views/composite/composite-view-include-context.json @@ -80,7 +80,7 @@ }, "dynamic": false }, - "query": "prefix music: prefix nxv: CONSTRUCT { {resource_id} music:name ?bandName ; music:genre ?bandGenre ; music:album ?albumId . ?albumId music:released ?albumReleaseDate ; music:song ?songId . ?songId music:title ?songTitle ; music:number ?songNumber ; music:length ?songLength } WHERE { {resource_id} music:name ?bandName ; music:genre ?bandGenre . OPTIONAL { {resource_id} ^music:by ?albumId . ?albumId music:released ?albumReleaseDate . OPTIONAL {?albumId ^music:on ?songId . ?songId music:title ?songTitle ; music:number ?songNumber ; music:length ?songLength } } } ORDER BY(?songNumber)", + "query": "{{bandQuery}}", "context": { "@base": "https://music.example.com/", "@vocab": "https://music.example.com/" @@ -121,7 +121,7 @@ }, "dynamic": false }, - "query": "prefix xsd: prefix music: prefix nxv: CONSTRUCT { {resource_id} music:name ?albumTitle ; music:length ?albumLength ; music:numberOfSongs ?numberOfSongs } WHERE {SELECT ?albumReleaseDate ?albumTitle (sum(xsd:integer(?songLength)) as ?albumLength) (count(?albumReleaseDate) as ?numberOfSongs) WHERE {OPTIONAL { {resource_id} ^music:on / music:length ?songLength } {resource_id} music:released ?albumReleaseDate ; music:title ?albumTitle . } GROUP BY ?albumReleaseDate ?albumTitle }", + "query": "{{albumQuery}}", "context": { "@base": "https://music.example.com/", "@vocab": "https://music.example.com/" diff --git a/tests/src/test/resources/kg/views/composite/composite-view.json b/tests/src/test/resources/kg/views/composite/composite-view.json index d9da4e0983..58a1e0b9b8 100644 --- a/tests/src/test/resources/kg/views/composite/composite-view.json +++ b/tests/src/test/resources/kg/views/composite/composite-view.json @@ -80,7 +80,7 @@ }, "dynamic": false }, - "query": "prefix music: prefix nxv: CONSTRUCT { {resource_id} music:name ?bandName ; music:genre ?bandGenre ; music:album ?albumId . ?albumId music:released ?albumReleaseDate ; music:song ?songId . ?songId music:title ?songTitle ; music:number ?songNumber ; music:length ?songLength } WHERE { {resource_id} music:name ?bandName ; music:genre ?bandGenre . OPTIONAL { {resource_id} ^music:by ?albumId . ?albumId music:released ?albumReleaseDate . OPTIONAL {?albumId ^music:on ?songId . ?songId music:title ?songTitle ; music:number ?songNumber ; music:length ?songLength } } } ORDER BY(?songNumber)", + "query": "{{bandQuery}}", "context": { "@base": "https://music.example.com/", "@vocab": "https://music.example.com/" @@ -120,7 +120,7 @@ }, "dynamic": false }, - "query": "prefix xsd: prefix music: prefix nxv: CONSTRUCT { {resource_id} music:name ?albumTitle ; music:length ?albumLength ; music:numberOfSongs ?numberOfSongs } WHERE {SELECT ?albumReleaseDate ?albumTitle (sum(xsd:integer(?songLength)) as ?albumLength) (count(?albumReleaseDate) as ?numberOfSongs) WHERE {OPTIONAL { {resource_id} ^music:on / music:length ?songLength } {resource_id} music:released ?albumReleaseDate ; music:title ?albumTitle . } GROUP BY ?albumReleaseDate ?albumTitle }", + "query": "{{albumQuery}}", "context": { "@base": "https://music.example.com/", "@vocab": "https://music.example.com/" diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala index b14a4bfbcc..32bd61d04c 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CompositeViewsSpec.scala @@ -7,11 +7,13 @@ import ch.epfl.bluebrain.nexus.tests.HttpClient._ import ch.epfl.bluebrain.nexus.tests.Identity.compositeviews.Jerry import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Events, Organizations, Views} +import ch.epfl.bluebrain.nexus.tests.kg.CompositeViewsSpec.{albumQuery, bandQuery} import com.typesafe.scalalogging.Logger import io.circe.Json import io.circe.optics.JsonPath._ import monix.bio.Task import monix.execution.Scheduler.Implicits.global + import scala.concurrent.duration._ class CompositeViewsSpec extends BaseSpec { @@ -136,7 +138,9 @@ class CompositeViewsSpec extends BaseSpec { "org" -> orgId, "org2" -> orgId, "remoteEndpoint" -> "http://delta:8080/v1", - "token" -> jerryToken + "token" -> jerryToken, + "bandQuery" -> bandQuery, + "albumQuery" -> albumQuery ): _* ) @@ -166,7 +170,9 @@ class CompositeViewsSpec extends BaseSpec { "org" -> orgId, "org2" -> orgId, "remoteEndpoint" -> "http://delta:8080/v1/other", - "token" -> jerryToken + "token" -> jerryToken, + "bandQuery" -> bandQuery, + "albumQuery" -> albumQuery ): _* ) @@ -183,7 +189,9 @@ class CompositeViewsSpec extends BaseSpec { "org" -> orgId, "org2" -> orgId, "remoteEndpoint" -> "http://delta:8080/v1", - "token" -> s"${jerryToken}wrong" + "token" -> s"${jerryToken}wrong", + "bandQuery" -> bandQuery, + "albumQuery" -> albumQuery ): _* ) @@ -207,7 +215,9 @@ class CompositeViewsSpec extends BaseSpec { "org" -> orgId, "org2" -> orgId, "remoteEndpoint" -> "http://fail.does.not.exist.at.all.asndkajbskhabsdfjhabsdfjkh/v1", - "token" -> jerryToken + "token" -> jerryToken, + "bandQuery" -> bandQuery, + "albumQuery" -> albumQuery ): _* ) @@ -312,7 +322,9 @@ class CompositeViewsSpec extends BaseSpec { "org" -> orgId, "org2" -> orgId, "remoteEndpoint" -> "http://delta:8080/v1", - "token" -> jerryToken + "token" -> jerryToken, + "bandQuery" -> bandQuery, + "albumQuery" -> albumQuery ): _* ) @@ -390,4 +402,75 @@ class CompositeViewsSpec extends BaseSpec { } } } + +} + +object CompositeViewsSpec { + + private val bandQuery = + raw""" + |PREFIX nxv: + |PREFIX music: + | + |CONSTRUCT + | { + | ?alias music:name ?bandName ; + | music:genre ?bandGenre ; + | music:album ?albumId . + | ?albumId music:released ?albumReleaseDate ; + | music:song ?songId . + | ?songId music:title ?songTitle ; + | music:number ?songNumber ; + | music:length ?songLength . + | } + |WHERE + | { VALUES ?id { {resource_id} } + | BIND(IRI(concat(str(?id), '/alias')) AS ?alias) + | + | ?id music:name ?bandName ; + | music:genre ?bandGenre + | + | OPTIONAL + | { ?id ^music:by ?albumId . + | ?albumId music:released ?albumReleaseDate + | OPTIONAL + | { ?albumId ^music:on ?songId . + | ?songId music:title ?songTitle ; + | music:number ?songNumber ; + | music:length ?songLength + | } + | } + | } + |ORDER BY ?songNumber + |""".stripMargin + .replaceAll("\\n", " ") + + private val albumQuery = + raw""" + |PREFIX xsd: + |PREFIX music: + |PREFIX nxv: + | + |CONSTRUCT + | { + | ?alias music:name ?albumTitle ; + | music:length ?albumLength ; + | music:numberOfSongs ?numberOfSongs . + | } + |WHERE + | { { SELECT ?id ?albumReleaseDate ?albumTitle (SUM(xsd:integer(?songLength)) AS ?albumLength) (COUNT(?albumReleaseDate) AS ?numberOfSongs) + | WHERE + | { VALUES ?id { {resource_id} } . + | OPTIONAL + | { ?id ^music:on/music:length ?songLength } + | ?id music:released ?albumReleaseDate ; + | music:title ?albumTitle . + | } + | GROUP BY ?id ?albumReleaseDate ?albumTitle + | } + | BIND(IRI(concat(str(?id), '/alias')) AS ?alias) + | } + |""".stripMargin + .replaceAll("\\n", " ") + } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala index 57ba062754..aaab7349c9 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala @@ -89,6 +89,7 @@ class SearchConfigSpec extends BaseSpec { json""" { "project" : { + "@id": "http://delta:8080/v1/projects/$id1", "identifier" : "http://delta:8080/v1/projects/$id1", "label" : "$id1" } @@ -106,6 +107,7 @@ class SearchConfigSpec extends BaseSpec { json""" { "license" : { + "@id" : "https://bbp.epfl.ch/neurosciencegraph/data/licenses/97521f71-605d-4f42-8f1b-c37e742a30bf", "identifier" : "https://bbp.epfl.ch/neurosciencegraph/data/licenses/97521f71-605d-4f42-8f1b-c37e742a30bf", "label" : "SSCX Portal Data Licence final v1.0" } From 81b26fcbf6fd10dc879d9d63103fb89c73680263 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 4 Aug 2023 17:43:08 +0200 Subject: [PATCH 2/3] Update Keycloak version in integration tests (#4137) Co-authored-by: Simon Dumas --- tests/docker/docker-compose.yml | 2 +- .../scala/ch/epfl/bluebrain/nexus/tests/KeycloakDsl.scala | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index 55d560fbc6..0e81868f73 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -60,7 +60,7 @@ services: # - /tmp:/default-volume keycloak: - image: quay.io/keycloak/keycloak:18.0.0 + image: quay.io/keycloak/keycloak:22.0.1 environment: KEYCLOAK_ADMIN: "admin" KEYCLOAK_ADMIN_PASSWORD: "admin" diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/KeycloakDsl.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/KeycloakDsl.scala index 6f18d93146..a0af601df5 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/KeycloakDsl.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/KeycloakDsl.scala @@ -74,9 +74,10 @@ class KeycloakDsl(implicit as: ActorSystem, materializer: Materializer, um: From def userToken(user: UserCredentials, client: ClientCredentials): Task[String] = { logger.info(s"Getting token for user ${user.name} for ${user.realm.name}") val clientFields = if (client.secret == "") { - Map("client_id" -> client.id) + Map("scope" -> "openid", "client_id" -> client.id) } else { Map( + "scope" -> "openid", "client_id" -> client.id, "client_secret" -> client.secret ) @@ -125,6 +126,7 @@ class KeycloakDsl(implicit as: ActorSystem, materializer: Materializer, um: From entity = akka.http.scaladsl.model .FormData( Map( + "scope" -> "openid", "grant_type" -> "client_credentials" ) ) From 221b199cb29dd042cb23fbc1c2c6bf993c061f75 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 7 Aug 2023 10:09:07 +0200 Subject: [PATCH 3/3] Add a multi-fetch operation (#4132) * Add a multi-fetch operation --------- Co-authored-by: Simon Dumas --- .../nexus/delta/routes/MultiFetchRoutes.scala | 48 +++++++ .../nexus/delta/wiring/DeltaModule.scala | 1 + .../nexus/delta/wiring/MultiFetchModule.scala | 47 +++++++ .../multi-fetch/all-unauthorized.json | 32 +++++ .../multi-fetch/compacted-response.json | 52 ++++++++ .../multi-fetch/source-response.json | 35 ++++++ .../delta/routes/MultiFetchRoutesSpec.scala | 100 +++++++++++++++ .../plugins/archive/ArchiveDownload.scala | 6 +- .../archive/model/ArchiveReference.scala | 21 ++-- .../model/ArchiveResourceRepresentation.scala | 92 -------------- .../plugins/archive/model/ArchiveState.scala | 12 +- .../plugins/archive/ArchiveDownloadSpec.scala | 2 +- .../archive/ArchivesDecodingSpec.scala | 2 +- .../plugins/archive/ArchivesSTMSpec.scala | 2 +- .../model/ArchiveSerializationSuite.scala | 3 +- .../sdk/directives/DeltaDirectives.scala | 2 +- .../sdk/model/ResourceRepresentation.scala | 99 +++++++++++++++ .../delta/sdk/multifetch/MultiFetch.scala | 51 ++++++++ .../multifetch/model/MultiFetchRequest.scala | 36 ++++++ .../multifetch/model/MultiFetchResponse.scala | 118 ++++++++++++++++++ .../delta/sdk/generators/ResourceGen.scala | 6 + .../sdk/multifetch/MultiFetchSuite.scala | 86 +++++++++++++ .../paradox/docs/delta/api/archives-api.md | 3 +- .../delta/api/assets/multi-fetch/payload.json | 17 +++ .../delta/api/assets/multi-fetch/request.sh | 21 ++++ .../api/assets/multi-fetch/response.json | 35 ++++++ docs/src/main/paradox/docs/delta/api/index.md | 1 + .../paradox/docs/delta/api/multi-fetch.md | 58 +++++++++ docs/src/main/paradox/docs/releases/index.md | 3 +- .../docs/releases/v1.9-release-notes.md | 6 + .../resources/kg/multi-fetch/all-success.json | 40 ++++++ .../kg/multi-fetch/limited-access.json | 32 +++++ .../resources/kg/multi-fetch/unknown.json | 13 ++ .../bluebrain/nexus/tests/HttpClient.scala | 5 + .../epfl/bluebrain/nexus/tests/Optics.scala | 15 ++- .../nexus/tests/kg/MultiFetchSpec.scala | 112 +++++++++++++++++ 36 files changed, 1091 insertions(+), 123 deletions(-) create mode 100644 delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutes.scala create mode 100644 delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/MultiFetchModule.scala create mode 100644 delta/app/src/test/resources/multi-fetch/all-unauthorized.json create mode 100644 delta/app/src/test/resources/multi-fetch/compacted-response.json create mode 100644 delta/app/src/test/resources/multi-fetch/source-response.json create mode 100644 delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutesSpec.scala delete mode 100644 delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveResourceRepresentation.scala create mode 100644 delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceRepresentation.scala create mode 100644 delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetch.scala create mode 100644 delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchRequest.scala create mode 100644 delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchResponse.scala create mode 100644 delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetchSuite.scala create mode 100644 docs/src/main/paradox/docs/delta/api/assets/multi-fetch/payload.json create mode 100644 docs/src/main/paradox/docs/delta/api/assets/multi-fetch/request.sh create mode 100644 docs/src/main/paradox/docs/delta/api/assets/multi-fetch/response.json create mode 100644 docs/src/main/paradox/docs/delta/api/multi-fetch.md create mode 100644 tests/src/test/resources/kg/multi-fetch/all-success.json create mode 100644 tests/src/test/resources/kg/multi-fetch/limited-access.json create mode 100644 tests/src/test/resources/kg/multi-fetch/unknown.json create mode 100644 tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutes.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutes.scala new file mode 100644 index 0000000000..9af9590cb5 --- /dev/null +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutes.scala @@ -0,0 +1,48 @@ +package ch.epfl.bluebrain.nexus.delta.routes + +import akka.http.scaladsl.server.Directives._ +import akka.http.scaladsl.server.Route +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution +import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.circe.CirceUnmarshalling +import ch.epfl.bluebrain.nexus.delta.sdk.directives.AuthDirectives +import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives._ +import ch.epfl.bluebrain.nexus.delta.sdk.directives.UriDirectives.baseUriPrefix +import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities +import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.RdfMarshalling +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.MultiFetch +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchRequest +import monix.execution.Scheduler + +/** + * Route allowing to fetch multiple resources in a single request + */ +class MultiFetchRoutes( + identities: Identities, + aclCheck: AclCheck, + multiFetch: MultiFetch +)(implicit + baseUri: BaseUri, + cr: RemoteContextResolution, + ordering: JsonKeyOrdering, + s: Scheduler +) extends AuthDirectives(identities, aclCheck) + with CirceUnmarshalling + with RdfMarshalling { + + def routes: Route = + baseUriPrefix(baseUri.prefix) { + pathPrefix("multi-fetch") { + pathPrefix("resources") { + extractCaller { implicit caller => + (get & entity(as[MultiFetchRequest])) { request => + emit(multiFetch(request).flatMap(_.asJson)) + } + } + } + } + } + +} diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala index 8a7767d41e..7c4f0915af 100644 --- a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala @@ -165,6 +165,7 @@ class DeltaModule(appCfg: AppConfig, config: Config)(implicit classLoader: Class include(ResolversModule) include(SchemasModule) include(ResourcesModule) + include(MultiFetchModule) include(IdentitiesModule) include(VersionModule) include(QuotasModule) diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/MultiFetchModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/MultiFetchModule.scala new file mode 100644 index 0000000000..dae425ae9a --- /dev/null +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/MultiFetchModule.scala @@ -0,0 +1,47 @@ +package ch.epfl.bluebrain.nexus.delta.wiring + +import ch.epfl.bluebrain.nexus.delta.Main.pluginsMaxPriority +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution +import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering +import ch.epfl.bluebrain.nexus.delta.routes.MultiFetchRoutes +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.MultiFetch +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchRequest +import ch.epfl.bluebrain.nexus.delta.sdk.{PriorityRoute, ResourceShifts} +import distage.ModuleDef +import izumi.distage.model.definition.Id +import monix.execution.Scheduler + +object MultiFetchModule extends ModuleDef { + + make[MultiFetch].from { + ( + aclCheck: AclCheck, + shifts: ResourceShifts + ) => + MultiFetch( + aclCheck, + (input: MultiFetchRequest.Input) => shifts.fetch(input.id, input.project) + ) + } + + make[MultiFetchRoutes].from { + ( + identities: Identities, + aclCheck: AclCheck, + multiFetch: MultiFetch, + baseUri: BaseUri, + rcr: RemoteContextResolution @Id("aggregate"), + jko: JsonKeyOrdering, + sc: Scheduler + ) => + new MultiFetchRoutes(identities, aclCheck, multiFetch)(baseUri, rcr, jko, sc) + } + + many[PriorityRoute].add { (route: MultiFetchRoutes) => + PriorityRoute(pluginsMaxPriority + 13, route.routes, requiresStrictEntity = true) + } + +} diff --git a/delta/app/src/test/resources/multi-fetch/all-unauthorized.json b/delta/app/src/test/resources/multi-fetch/all-unauthorized.json new file mode 100644 index 0000000000..e998afff24 --- /dev/null +++ b/delta/app/src/test/resources/multi-fetch/all-unauthorized.json @@ -0,0 +1,32 @@ +{ + "format": "compacted", + "resources": [ + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/success", + "error": { + "@context": "https://bluebrain.github.io/nexus/contexts/error.json", + "@type": "AuthorizationFailed", + "reason": "The supplied authentication is not authorized to access this resource." + }, + "project": "org/proj1" + }, + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/not-found", + "error": { + "@context": "https://bluebrain.github.io/nexus/contexts/error.json", + "@type": "AuthorizationFailed", + "reason": "The supplied authentication is not authorized to access this resource." + }, + "project": "org/proj1" + }, + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/unauthorized", + "error": { + "@context": "https://bluebrain.github.io/nexus/contexts/error.json", + "@type": "AuthorizationFailed", + "reason": "The supplied authentication is not authorized to access this resource." + }, + "project": "org/proj2" + } + ] +} \ No newline at end of file diff --git a/delta/app/src/test/resources/multi-fetch/compacted-response.json b/delta/app/src/test/resources/multi-fetch/compacted-response.json new file mode 100644 index 0000000000..e7dc0a0333 --- /dev/null +++ b/delta/app/src/test/resources/multi-fetch/compacted-response.json @@ -0,0 +1,52 @@ +{ + "format": "compacted", + "resources": [ + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/success", + "project": "org/proj1", + "value": { + "@context": [ + { + "@vocab": "https://bluebrain.github.io/nexus/vocabulary/" + }, + "https://bluebrain.github.io/nexus/contexts/metadata.json" + ], + "@id": "https://bluebrain.github.io/nexus/vocabulary/success", + "@type": "Custom", + "bool": false, + "name": "Alex", + "number": 24, + "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/unconstrained.json", + "_createdAt": "1970-01-01T00:00:00Z", + "_createdBy": "http://localhost/v1/anonymous", + "_deprecated": false, + "_incoming": "http://localhost/v1/resources/org/proj1/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fschemas%2Funconstrained.json/success/incoming", + "_outgoing": "http://localhost/v1/resources/org/proj1/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fschemas%2Funconstrained.json/success/outgoing", + "_project": "http://localhost/v1/projects/org/proj1", + "_rev": 1, + "_schemaProject": "http://localhost/v1/projects/org/proj1", + "_self": "http://localhost/v1/resources/org/proj1/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fschemas%2Funconstrained.json/success", + "_updatedAt": "1970-01-01T00:00:00Z", + "_updatedBy": "http://localhost/v1/anonymous" + } + }, + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/not-found", + "error": { + "@context": "https://bluebrain.github.io/nexus/contexts/error.json", + "@type": "NotFound", + "reason": "The resource 'https://bluebrain.github.io/nexus/vocabulary/not-found' was not found in project 'org/proj1'." + }, + "project": "org/proj1" + }, + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/unauthorized", + "error": { + "@context": "https://bluebrain.github.io/nexus/contexts/error.json", + "@type": "AuthorizationFailed", + "reason": "The supplied authentication is not authorized to access this resource." + }, + "project": "org/proj2" + } + ] +} \ No newline at end of file diff --git a/delta/app/src/test/resources/multi-fetch/source-response.json b/delta/app/src/test/resources/multi-fetch/source-response.json new file mode 100644 index 0000000000..ee5bace34d --- /dev/null +++ b/delta/app/src/test/resources/multi-fetch/source-response.json @@ -0,0 +1,35 @@ +{ + "format": "source", + "resources": [ + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/success", + "project": "org/proj1", + "value": { + "@context": { + "@vocab": "https://bluebrain.github.io/nexus/vocabulary/" + }, + "@id": "https://bluebrain.github.io/nexus/vocabulary/success", + "@type": "Custom", + "bool": false, + "name": "Alex", + "number": 24 + } + }, + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/not-found", + "error": { + "@type": "NotFound", + "reason": "The resource 'https://bluebrain.github.io/nexus/vocabulary/not-found' was not found in project 'org/proj1'." + }, + "project": "org/proj1" + }, + { + "@id": "https://bluebrain.github.io/nexus/vocabulary/unauthorized", + "error": { + "@type": "AuthorizationFailed", + "reason": "The supplied authentication is not authorized to access this resource." + }, + "project": "org/proj2" + } + ] +} \ No newline at end of file diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutesSpec.scala new file mode 100644 index 0000000000..7e64b16f90 --- /dev/null +++ b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/MultiFetchRoutesSpec.scala @@ -0,0 +1,100 @@ +package ch.epfl.bluebrain.nexus.delta.routes + +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.headers.OAuth2BearerToken +import akka.http.scaladsl.server.Route +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck +import ch.epfl.bluebrain.nexus.delta.sdk.generators.ResourceGen +import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.MultiFetch +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchRequest +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions +import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest +import monix.bio.UIO + +class MultiFetchRoutesSpec extends BaseRouteSpec { + + implicit private val caller: Caller = + Caller(alice, Set(alice, Anonymous, Authenticated(realm), Group("group", realm))) + + private val asAlice = addCredentials(OAuth2BearerToken("alice")) + + private val identities = IdentitiesDummy(caller) + + private val project1 = ProjectRef.unsafe("org", "proj1") + private val project2 = ProjectRef.unsafe("org", "proj2") + + private val permissions = Set(Permissions.resources.read) + private val aclCheck = AclSimpleCheck((alice, project1, permissions)).runSyncUnsafe() + + private val successId = nxv + "success" + private val successContent = + ResourceGen.jsonLdContent(successId, project1, jsonContentOf("resources/resource.json", "id" -> successId)) + + private val notFoundId = nxv + "not-found" + private val unauthorizedId = nxv + "unauthorized" + + private def fetchResource = + (input: MultiFetchRequest.Input) => { + input match { + case MultiFetchRequest.Input(Latest(`successId`), `project1`) => + UIO.some(successContent) + case _ => UIO.none + } + } + + private val multiFetch = MultiFetch( + aclCheck, + fetchResource + ) + + private val routes = Route.seal( + new MultiFetchRoutes(identities, aclCheck, multiFetch).routes + ) + + "The Multi fetch route" should { + + val endpoint = "/v1/multi-fetch/resources" + + def request(format: ResourceRepresentation) = + json""" + { + "format": "$format", + "resources": [ + { "id": "$successId", "project": "$project1" }, + { "id": "$notFoundId", "project": "$project1" }, + { "id": "$unauthorizedId", "project": "$project2" } + ] + }""" + + "return unauthorised results for a user with no access" in { + val entity = request(ResourceRepresentation.CompactedJsonLd).toEntity + Get(endpoint, entity) ~> routes ~> check { + status shouldEqual StatusCodes.OK + response.asJson shouldEqual jsonContentOf("multi-fetch/all-unauthorized.json") + } + } + + "return expected results as compacted json-ld for a user with limited access" in { + val entity = request(ResourceRepresentation.CompactedJsonLd).toEntity + Get(endpoint, entity) ~> asAlice ~> routes ~> check { + status shouldEqual StatusCodes.OK + response.asJson shouldEqual jsonContentOf("multi-fetch/compacted-response.json") + } + } + + "return expected results as original payloads for a user with limited access" in { + val entity = request(ResourceRepresentation.SourceJson).toEntity + Get(endpoint, entity) ~> asAlice ~> routes ~> check { + status shouldEqual StatusCodes.OK + response.asJson shouldEqual jsonContentOf("multi-fetch/source-response.json") + } + } + } +} diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala index eac82d552f..30f14af772 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala +++ b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala @@ -5,7 +5,7 @@ import akka.util.ByteString import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{FileReference, FileSelfReference, ResourceReference} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveResourceRepresentation._ +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation._ import ch.epfl.bluebrain.nexus.delta.plugins.archive.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection import ch.epfl.bluebrain.nexus.delta.rdf.RdfError @@ -21,7 +21,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.directives.Response.Complete import ch.epfl.bluebrain.nexus.delta.sdk.error.SDKError import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent -import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri +import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceRepresentation} import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources import ch.epfl.bluebrain.nexus.delta.sdk.stream.StreamConverter import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, JsonLdValue} @@ -249,7 +249,7 @@ object ArchiveDownload { private def valueToByteString[A]( value: JsonLdContent[A, _], - repr: ArchiveResourceRepresentation + repr: ResourceRepresentation ): IO[RdfError, ByteString] = { implicit val encoder: JsonLdEncoder[A] = value.encoder repr match { diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveReference.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveReference.scala index e4d41d0f8c..d26ffc7a44 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveReference.scala +++ b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveReference.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.archive.model import akka.http.scaladsl.model.Uri import cats.Order import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils -import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveResourceRepresentation.{CompactedJsonLd, SourceJson} +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation.{CompactedJsonLd, SourceJson} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv @@ -12,6 +12,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoderError.ParsingFailure import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.configuration.semiauto._ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.{Configuration, JsonLdDecoder} +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.{Latest, Revision, Tag} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} @@ -68,10 +69,10 @@ object ArchiveReference { ref: ResourceRef, project: Option[ProjectRef], path: Option[AbsolutePath], - representation: Option[ArchiveResourceRepresentation] + representation: Option[ResourceRepresentation] ) extends FullArchiveReference { - def representationOrDefault: ArchiveResourceRepresentation = representation.getOrElse(CompactedJsonLd) + def representationOrDefault: ResourceRepresentation = representation.getOrElse(CompactedJsonLd) def defaultFileName = s"${UrlUtils.encode(ref.original.toString)}${representationOrDefault.extension}" } @@ -132,7 +133,7 @@ object ArchiveReference { rev: Option[Int], path: Option[AbsolutePath], originalSource: Option[Boolean], - format: Option[ArchiveResourceRepresentation] + format: Option[ResourceRepresentation] ) extends ReferenceInput final private case class FileInput( @@ -165,11 +166,11 @@ object ArchiveReference { implicit val cfg: Configuration = Configuration.default.copy(context = ctx) deriveConfigJsonLdDecoder[ReferenceInput].flatMap { - case ResourceInput(_, _, Some(_: UserTag), Some(_: Int), _, _, _) => + case ResourceInput(_, _, Some(_: UserTag), Some(_: Int), _, _, _) => Left(ParsingFailure("An archive resource reference cannot use both 'rev' and 'tag' fields.")) - case ResourceInput(_, _, _, _, _, Some(_: Boolean), Some(_: ArchiveResourceRepresentation)) => + case ResourceInput(_, _, _, _, _, Some(_: Boolean), Some(_: ResourceRepresentation)) => Left(ParsingFailure("An archive resource reference cannot use both 'originalSource' and 'format' fields.")) - case ResourceInput(resourceId, project, tag, rev, path, originalSource, format) => + case ResourceInput(resourceId, project, tag, rev, path, originalSource, format) => val ref = refOf(resourceId, tag, rev) val repr = (originalSource, format) match { case (_, Some(repr)) => Some(repr) @@ -178,12 +179,12 @@ object ArchiveReference { case _ => None } Right(ResourceReference(ref, project, path, repr)) - case FileInput(_, _, Some(_: UserTag), Some(_: Int), _) => + case FileInput(_, _, Some(_: UserTag), Some(_: Int), _) => Left(ParsingFailure("An archive file reference cannot use both 'rev' and 'tag' fields.")) - case FileInput(resourceId, project, tag, rev, path) => + case FileInput(resourceId, project, tag, rev, path) => val ref = refOf(resourceId, tag, rev) Right(FileReference(ref, project, path)) - case FileSelfInput(value, path) => + case FileSelfInput(value, path) => Right(FileSelfReference(value, path)) } } diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveResourceRepresentation.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveResourceRepresentation.scala deleted file mode 100644 index d140c59e26..0000000000 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveResourceRepresentation.scala +++ /dev/null @@ -1,92 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.archive.model - -import cats.Order -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoder -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoderError.ParsingFailure -import io.circe.Encoder - -/** - * Enumeration of representations for resource references. - */ -sealed trait ArchiveResourceRepresentation extends Product with Serializable { - - /** - * Default extension for the format - */ - def extension: String - -} - -object ArchiveResourceRepresentation { - - /** - * Source representation of a resource. - */ - final case object SourceJson extends ArchiveResourceRepresentation { - override def extension: String = ".json" - - override val toString: String = "source" - } - - /** - * Compacted JsonLD representation of a resource. - */ - final case object CompactedJsonLd extends ArchiveResourceRepresentation { - override def extension: String = ".json" - - override val toString: String = "compacted" - } - - /** - * Expanded JsonLD representation of a resource. - */ - final case object ExpandedJsonLd extends ArchiveResourceRepresentation { - override def extension: String = ".json" - - override val toString: String = "expanded" - } - - /** - * NTriples representation of a resource. - */ - final case object NTriples extends ArchiveResourceRepresentation { - override def extension: String = ".nt" - - override val toString: String = "n-triples" - } - - final case object NQuads extends ArchiveResourceRepresentation { - override def extension: String = ".nq" - - override val toString: String = "n-quads" - } - - /** - * Dot representation of a resource. - */ - final case object Dot extends ArchiveResourceRepresentation { - override def extension: String = ".dot" - - override val toString: String = "dot" - } - - implicit final val archiveResourceRepresentationJsonLdDecoder: JsonLdDecoder[ArchiveResourceRepresentation] = - JsonLdDecoder.stringJsonLdDecoder.andThen { (cursor, str) => - str match { - case SourceJson.toString => Right(SourceJson) - case CompactedJsonLd.toString => Right(CompactedJsonLd) - case ExpandedJsonLd.toString => Right(ExpandedJsonLd) - case NTriples.toString => Right(NTriples) - case NQuads.toString => Right(NQuads) - case Dot.toString => Right(Dot) - case other => Left(ParsingFailure("Format", other, cursor.history)) - } - } - - implicit final val archiveResourceRepresentationEncoder: Encoder[ArchiveResourceRepresentation] = - Encoder.encodeString.contramap { - _.toString - } - - implicit val archiveResourceRepresentationOrder: Order[ArchiveResourceRepresentation] = Order.by(_.toString) -} diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveState.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveState.scala index 1c4f5b5db0..49d6b26768 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveState.scala +++ b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveState.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptySet import ch.epfl.bluebrain.nexus.delta.plugins.archive.model import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.instances._ -import ch.epfl.bluebrain.nexus.delta.sdk.model.{ResourceF, ResourceUris} +import ch.epfl.bluebrain.nexus.delta.sdk.model.{ResourceF, ResourceRepresentation, ResourceUris} import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectBase} import ch.epfl.bluebrain.nexus.delta.sourcing.Serializer import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject @@ -65,11 +65,11 @@ object ArchiveState { @nowarn("cat=unused") implicit val serializer: Serializer[Iri, ArchiveState] = { import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Database._ - implicit val configuration: Configuration = Serializer.circeConfiguration - implicit val archiveResourceRepresentation: Codec.AsObject[ArchiveResourceRepresentation] = - deriveConfiguredCodec[ArchiveResourceRepresentation] - implicit val archiveReferenceCodec: Codec.AsObject[ArchiveReference] = deriveConfiguredCodec[ArchiveReference] - implicit val codec: Codec.AsObject[ArchiveState] = deriveConfiguredCodec[ArchiveState] + implicit val configuration: Configuration = Serializer.circeConfiguration + implicit val archiveResourceRepresentation: Codec.AsObject[ResourceRepresentation] = + deriveConfiguredCodec[ResourceRepresentation] + implicit val archiveReferenceCodec: Codec.AsObject[ArchiveReference] = deriveConfiguredCodec[ArchiveReference] + implicit val codec: Codec.AsObject[ArchiveState] = deriveConfiguredCodec[ArchiveState] Serializer() } diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala index e5c64d3622..a549e747de 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala @@ -11,7 +11,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils.encode import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{FileReference, FileSelfReference, ResourceReference} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.{AuthorizationFailed, FilenameTooLong, InvalidFileSelf, ResourceNotFound} -import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveResourceRepresentation.{CompactedJsonLd, Dot, ExpandedJsonLd, NQuads, NTriples, SourceJson} +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation.{CompactedJsonLd, Dot, ExpandedJsonLd, NQuads, NTriples, SourceJson} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.{ArchiveFormat, ArchiveRejection, ArchiveValue} import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesDecodingSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesDecodingSpec.scala index b9414611b1..318d4d7226 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesDecodingSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesDecodingSpec.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptySet import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{FileReference, ResourceReference} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.{DecodingFailed, InvalidJsonLdFormat, UnexpectedArchiveId} -import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveResourceRepresentation.{CompactedJsonLd, Dot, ExpandedJsonLd, NTriples, SourceJson} +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation.{CompactedJsonLd, Dot, ExpandedJsonLd, NTriples, SourceJson} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.rdf.implicits._ diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesSTMSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesSTMSpec.scala index 4cbcc490a4..bed5a9c895 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesSTMSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivesSTMSpec.scala @@ -2,7 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.archive import cats.data.NonEmptySet import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.ResourceReference -import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveResourceRepresentation.SourceJson +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation.SourceJson import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.{ArchiveState, ArchiveValue, CreateArchive} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveSerializationSuite.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveSerializationSuite.scala index 676156a243..09c5a6f1f6 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveSerializationSuite.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveSerializationSuite.scala @@ -6,6 +6,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.SerializationSuite import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Subject, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} @@ -27,7 +28,7 @@ class ArchiveSerializationSuite extends SerializationSuite { ResourceRef.Revision(iri"$resourceId?rev=1", resourceId, 1), Some(anotherProject), absolutePath, - Some(ArchiveResourceRepresentation.CompactedJsonLd) + Some(ResourceRepresentation.CompactedJsonLd) ) private val fileSelfReference = FileSelfReference(uri"https://bbp.epfl.ch/nexus/org/proj/file", absolutePath) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala index 71c71486b4..6c5ad9c3fb 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala @@ -102,7 +102,7 @@ trait DeltaDirectives extends UriDirectives { def unacceptedMediaTypeRejection(values: Seq[MediaType]): UnacceptedResponseContentTypeRejection = UnacceptedResponseContentTypeRejection(values.map(mt => Alternative(mt)).toSet) - private[directives] def requestMediaType: Directive1[MediaType] = + def requestMediaType: Directive1[MediaType] = extractRequest.flatMap { req => HeadersUtils.findFirst(req.headers, mediaTypes) match { case Some(value) => provide(value) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceRepresentation.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceRepresentation.scala new file mode 100644 index 0000000000..5257c1b07c --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceRepresentation.scala @@ -0,0 +1,99 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.model + +import cats.Order +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoder +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoderError.ParsingFailure +import io.circe.{Decoder, Encoder} + +/** + * Enumeration of representations for resources. + */ +sealed trait ResourceRepresentation extends Product with Serializable { + + /** + * Default extension for the format + */ + def extension: String + +} + +object ResourceRepresentation { + + /** + * Source representation of a resource. + */ + final case object SourceJson extends ResourceRepresentation { + override def extension: String = ".json" + + override val toString: String = "source" + } + + /** + * Compacted JsonLD representation of a resource. + */ + final case object CompactedJsonLd extends ResourceRepresentation { + override def extension: String = ".json" + + override val toString: String = "compacted" + } + + /** + * Expanded JsonLD representation of a resource. + */ + final case object ExpandedJsonLd extends ResourceRepresentation { + override def extension: String = ".json" + + override val toString: String = "expanded" + } + + /** + * NTriples representation of a resource. + */ + final case object NTriples extends ResourceRepresentation { + override def extension: String = ".nt" + + override val toString: String = "n-triples" + } + + final case object NQuads extends ResourceRepresentation { + override def extension: String = ".nq" + + override val toString: String = "n-quads" + } + + /** + * Dot representation of a resource. + */ + final case object Dot extends ResourceRepresentation { + override def extension: String = ".dot" + + override val toString: String = "dot" + } + + private def parse(value: String) = + value match { + case SourceJson.toString => Right(SourceJson) + case CompactedJsonLd.toString => Right(CompactedJsonLd) + case ExpandedJsonLd.toString => Right(ExpandedJsonLd) + case NTriples.toString => Right(NTriples) + case NQuads.toString => Right(NQuads) + case Dot.toString => Right(Dot) + case other => Left(s"$other is not a valid representation") + } + + implicit final val resourceRepresentationJsonLdDecoder: JsonLdDecoder[ResourceRepresentation] = + JsonLdDecoder.stringJsonLdDecoder.andThen { (cursor, str) => + parse(str).leftMap(_ => ParsingFailure("Format", str, cursor.history)) + } + + implicit final val resourceRepresentationDecoder: Decoder[ResourceRepresentation] = + Decoder.decodeString.emap(parse) + + implicit final val resourceRepresentationEncoder: Encoder[ResourceRepresentation] = + Encoder.encodeString.contramap { + _.toString + } + + implicit val resourceRepresentationOrder: Order[ResourceRepresentation] = Order.by(_.toString) +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetch.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetch.scala new file mode 100644 index 0000000000..22bcbee928 --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetch.scala @@ -0,0 +1,51 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.multifetch + +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchResponse.Result._ +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.{MultiFetchRequest, MultiFetchResponse} +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources +import monix.bio.UIO + +/** + * Allows to fetch multiple resources of different types in one request. + * + * The response includes a resources array that contains the resources in the order specified in the request. If there + * is a failure getting a particular resource, the error is included in place of the resource. + */ +trait MultiFetch { + + def apply(request: MultiFetchRequest)(implicit caller: Caller): UIO[MultiFetchResponse] + +} + +object MultiFetch { + def apply( + aclCheck: AclCheck, + fetchResource: MultiFetchRequest.Input => UIO[Option[JsonLdContent[_, _]]] + ): MultiFetch = + new MultiFetch { + override def apply(request: MultiFetchRequest)(implicit + caller: Caller + ): UIO[MultiFetchResponse] = { + val fetchAllCached = aclCheck.fetchAll.memoizeOnSuccess + request.resources + .traverse { input => + aclCheck.authorizeFor(input.project, resources.read, fetchAllCached).flatMap { + case true => + fetchResource(input).map { + _.map(Success(input.id, input.project, _)) + .getOrElse(NotFound(input.id, input.project)) + } + case false => + UIO.pure(AuthorizationFailed(input.id, input.project)) + } + } + .map { resources => + MultiFetchResponse(request.format, resources) + } + } + + } +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchRequest.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchRequest.scala new file mode 100644 index 0000000000..48aba61ec4 --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchRequest.scala @@ -0,0 +1,36 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model + +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchRequest.Input +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import io.circe.Decoder + +import scala.annotation.nowarn + +/** + * Request to get multiple resources + * @param format + * the output format for these resources + * @param resources + * the list of resources + */ +final case class MultiFetchRequest(format: ResourceRepresentation, resources: NonEmptyList[Input]) {} + +object MultiFetchRequest { + + def apply(representation: ResourceRepresentation, first: Input, others: Input*) = + new MultiFetchRequest(representation, NonEmptyList.of(first, others: _*)) + + final case class Input(id: ResourceRef, project: ProjectRef) + + @nowarn("cat=unused") + implicit val multiFetchRequestDecoder: Decoder[MultiFetchRequest] = { + import io.circe.generic.extras.Configuration + import io.circe.generic.extras.semiauto._ + implicit val cfg: Configuration = Configuration.default + implicit val inputDecoder = deriveConfiguredDecoder[Input] + deriveConfiguredDecoder[MultiFetchRequest] + } + +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchResponse.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchResponse.scala new file mode 100644 index 0000000000..17ba74ed08 --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchResponse.scala @@ -0,0 +1,118 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model + +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.rdf.RdfError +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.contexts +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.jsonLdEncoderSyntax +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation.{CompactedJsonLd, Dot, ExpandedJsonLd, NQuads, NTriples, SourceJson} +import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceRepresentation} +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchResponse.Result +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchResponse.Result.itemEncoder +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import io.circe.syntax.EncoderOps +import io.circe.{Encoder, Json, JsonObject} +import monix.bio.{IO, UIO} + +/** + * A response for a multi-fetch operation + * @param format + * the formats in which the resource should be represented + * @param resources + * the result for each resource + */ +final case class MultiFetchResponse(format: ResourceRepresentation, resources: NonEmptyList[Result]) { + + /** + * Encode the response as a Json payload + */ + def asJson(implicit base: BaseUri, rcr: RemoteContextResolution): UIO[Json] = { + val encodeItem = itemEncoder(format) + resources.traverse(encodeItem).map { r => + Json.obj( + "format" -> format.asJson, + "resources" -> r.asJson + ) + } + }.hideErrors +} + +object MultiFetchResponse { + + sealed trait Result { + + def id: ResourceRef + + def project: ProjectRef + } + + object Result { + + sealed trait Error extends Result { + def reason: String + } + + final case class AuthorizationFailed(id: ResourceRef, project: ProjectRef) extends Error { + override def reason: String = "The supplied authentication is not authorized to access this resource." + } + + final case class NotFound(id: ResourceRef, project: ProjectRef) extends Error { + override def reason: String = s"The resource '${id.toString}' was not found in project '$project'." + } + + final case class Success[A](id: ResourceRef, project: ProjectRef, content: JsonLdContent[A, _]) extends Result + + implicit private val itemErrorEncoder: Encoder.AsObject[Error] = { + Encoder.AsObject.instance[Error] { r => + JsonObject( + "@type" -> Json.fromString(r.getClass.getSimpleName), + "reason" -> Json.fromString(r.reason) + ) + } + } + + implicit val itemErrorJsonLdEncoder: JsonLdEncoder[Error] = { + JsonLdEncoder.computeFromCirce(ContextValue(contexts.error)) + } + + implicit private val api: JsonLdApi = JsonLdJavaApi.lenient + + private[model] def itemEncoder(repr: ResourceRepresentation)(implicit base: BaseUri, rcr: RemoteContextResolution) = + (item: Result) => { + val common = JsonObject( + "@id" -> item.id.asJson, + "project" -> item.project.asJson + ) + + def valueToJson[A](value: JsonLdContent[A, _]): IO[RdfError, Json] = { + implicit val encoder: JsonLdEncoder[A] = value.encoder + toJson(value.resource, value.source) + } + + def toJson[C, S](value: C, source: S)(implicit + valueJsonLdEncoder: JsonLdEncoder[C], + sourceEncoder: Encoder[S] + ): IO[RdfError, Json] = + repr match { + case SourceJson => UIO.pure(source.asJson) + case CompactedJsonLd => value.toCompactedJsonLd.map { v => v.json } + case ExpandedJsonLd => value.toExpandedJsonLd.map { v => v.json } + case NTriples => value.toNTriples.map { v => v.value.asJson } + case NQuads => value.toNQuads.map { v => v.value.asJson } + case Dot => value.toDot.map { v => v.value.asJson } + } + + val result = item match { + case e: Error => toJson(e, e).map { e => JsonObject("error" -> e) } + case Success(_, _, content) => valueToJson(content).map { r => JsonObject("value" -> r) } + } + + result.map(_.deepMerge(common)) + } + + } + +} diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceGen.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceGen.scala index be576cc180..6ceb1ac7f3 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceGen.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceGen.scala @@ -6,6 +6,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.ExpandedJsonLd import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.DataResource +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent import ch.epfl.bluebrain.nexus.delta.sdk.model.Tags import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectBase} import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.{Resource, ResourceState} @@ -128,4 +129,9 @@ object ResourceGen extends IOValues { subject ).toResource(am, ProjectBase.unsafe(base)) + def jsonLdContent(id: Iri, project: ProjectRef, source: Json)(implicit resolution: RemoteContextResolution) = { + val resourceF = sourceToResourceF(id, project, source) + JsonLdContent(resourceF, resourceF.value.source, None) + } + } diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetchSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetchSuite.scala new file mode 100644 index 0000000000..84ba553549 --- /dev/null +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/MultiFetchSuite.scala @@ -0,0 +1,86 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.multifetch + +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck +import ch.epfl.bluebrain.nexus.delta.sdk.generators.ResourceGen +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceRepresentation +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchRequest.Input +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.MultiFetchResponse.Result.{AuthorizationFailed, NotFound, Success} +import ch.epfl.bluebrain.nexus.delta.sdk.multifetch.model.{MultiFetchRequest, MultiFetchResponse} +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions +import ch.epfl.bluebrain.nexus.delta.sdk.utils.Fixtures +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, Label, ProjectRef} +import ch.epfl.bluebrain.nexus.testkit.TestHelpers +import ch.epfl.bluebrain.nexus.testkit.bio.BioSuite +import monix.bio.UIO + +class MultiFetchSuite extends BioSuite with TestHelpers with Fixtures { + + implicit private val subject: Subject = Identity.User("user", Label.unsafe("realm")) + implicit private val caller: Caller = Caller.unsafe(subject) + + private val project1 = ProjectRef.unsafe("org", "proj1") + private val project2 = ProjectRef.unsafe("org", "proj2") + + private val permissions = Set(Permissions.resources.read) + private val aclCheck = AclSimpleCheck((subject, project1, permissions)).runSyncUnsafe() + + private val successId = nxv + "success" + private val successContent = + ResourceGen.jsonLdContent(successId, project1, jsonContentOf("resources/resource.json", "id" -> successId)) + private val notFoundId = nxv + "not-found" + private val unauthorizedId = nxv + "unauthorized" + + private def fetchResource = + (input: MultiFetchRequest.Input) => { + input match { + case MultiFetchRequest.Input(Latest(`successId`), `project1`) => + UIO.some(successContent) + case _ => UIO.none + } + } + + private val multiFetch = MultiFetch( + aclCheck, + fetchResource + ) + + private val request = MultiFetchRequest( + ResourceRepresentation.NTriples, + Input(Latest(successId), project1), + Input(Latest(notFoundId), project1), + Input(Latest(unauthorizedId), project2) + ) + + test("Return the response matching the user acls") { + + val expected = MultiFetchResponse( + ResourceRepresentation.NTriples, + NonEmptyList.of( + Success(Latest(successId), project1, successContent), + NotFound(Latest(notFoundId), project1), + AuthorizationFailed(Latest(unauthorizedId), project2) + ) + ) + + multiFetch(request).assert(expected) + } + + test("Return only unauthorized for a user with no access") { + val expected = MultiFetchResponse( + ResourceRepresentation.NTriples, + NonEmptyList.of( + AuthorizationFailed(Latest(successId), project1), + AuthorizationFailed(Latest(notFoundId), project1), + AuthorizationFailed(Latest(unauthorizedId), project2) + ) + ) + + multiFetch(request)(Caller.Anonymous).assert(expected) + } + +} diff --git a/docs/src/main/paradox/docs/delta/api/archives-api.md b/docs/src/main/paradox/docs/delta/api/archives-api.md index 80a7400395..30e9274b42 100644 --- a/docs/src/main/paradox/docs/delta/api/archives-api.md +++ b/docs/src/main/paradox/docs/delta/api/archives-api.md @@ -88,7 +88,8 @@ In order to decide whether we want to select a resource or a file, the `@type` d possibilities: - `Resource`: targets a resource -- `File`: targets a file +- `File`: targets a file using its project and id +- `FileSelf`: targets a file using its address (`_self`) ## Create using POST diff --git a/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/payload.json b/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/payload.json new file mode 100644 index 0000000000..7eb1afcaac --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/payload.json @@ -0,0 +1,17 @@ +{ + "format": "source", + "resources" : [ + { + "id": "https://bbp.epfl.ch/person/alex", + "project": "public/person" + }, + { + "id": "https://bbp.epfl.ch/person/john-doe", + "project": "public/person" + }, + { + "id": "https://bbp.epfl.ch/secret/xxx", + "project": "restricted/xxx" + } + ] +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/request.sh b/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/request.sh new file mode 100644 index 0000000000..1ec947ab42 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/request.sh @@ -0,0 +1,21 @@ +curl -L \ + -X GET \ + -d ' + { + "format": "source", + "resources" : [ + { + "id": "https://bbp.epfl.ch/person/alex", + "project": "public/person" + }, + { + "id": "https://bbp.epfl.ch/person/john-doe", + "project": "public/person" + }, + { + "id": "https://bbp.epfl.ch/secret/xxx", + "project": "restricted/xxx" + } + ] + } +' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/response.json b/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/response.json new file mode 100644 index 0000000000..327120d043 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/multi-fetch/response.json @@ -0,0 +1,35 @@ +{ + "format": "source", + "resources": [ + { + "@id": "https://bbp.epfl.ch/person/alex", + "project": "public/person", + "value": { + "@context": { + "@vocab": "https://bluebrain.github.io/nexus/vocabulary/" + }, + "@id": "https://bluebrain.github.io/nexus/vocabulary/success", + "@type": "Person", + "bool": false, + "name": "Alex", + "number": 24 + } + }, + { + "@id": "https://bbp.epfl.ch/person/john-doe", + "project": "public/person", + "error": { + "@type": "NotFound", + "reason": "The resource 'https://bbp.epfl.ch/person/john-doe' was not found in project 'public/person'." + } + }, + { + "@id": "https://bbp.epfl.ch/secret/xxx", + "project": "restricted/xxx", + "error": { + "@type": "AuthorizationFailed", + "reason": "The supplied authentication is not authorized to access this resource." + } + } + ] +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/index.md b/docs/src/main/paradox/docs/delta/api/index.md index f5fba971d5..57ab90a186 100644 --- a/docs/src/main/paradox/docs/delta/api/index.md +++ b/docs/src/main/paradox/docs/delta/api/index.md @@ -13,6 +13,7 @@ * @ref:[Quotas](quotas.md) * @ref:[Schemas](schemas-api.md) * @ref:[Resources](resources-api.md) +* @ref:[Multi-fetch](multi-fetch.md) * @ref:[Resolvers](resolvers-api.md) * @ref:[Views](views/index.md) * @ref:[Storages](storages-api.md) diff --git a/docs/src/main/paradox/docs/delta/api/multi-fetch.md b/docs/src/main/paradox/docs/delta/api/multi-fetch.md new file mode 100644 index 0000000000..4288a829a7 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/multi-fetch.md @@ -0,0 +1,58 @@ +# Multi fetch + +The multi-fetch operation allows to get in a given format multiple resources that can live in multiple projects. + +The response includes a resources array that contains the resources in the order specified in the request. +The structure of the returned resources is similar to that returned by the fetch API. +If there is a failure getting a particular resource, the error is included in place of the resource. + +This operation can be used to return every type of resource. + +@@@ note { .tip title="Authorization notes" } + +When performing a request, the caller must have `resources/read` permission on the project each resource belongs to. + +Please visit @ref:[Authentication & authorization](authentication.md) section to learn more about it. + +@@@ + +## Payload + +``` +GET /v1/multi-fetch/resources + +{ + "format": {format} + "resources": [ + { + "id": "{id}", + "project": "{project}" + }, + ... + ] +} +``` + +where... + +- `{format}`: String - the format we expect for the resources in the response. +Accepts the following values: source (to get the original payload), compacted, expanded, n-triples, dot +- `{project}`: String - the project (in the format 'myorg/myproject') where the specified resource belongs. This field + is optional. It defaults to the current project. +- `{id}`: Iri - the @id value of the resource to be returned. Can contain a tag or a revision. + +## Example + +The following example shows how to perform a multi-fetch and an example of response +containing errors (missing permissions and resource not found). +As a response, a regular json is returned containing the different resources in the requested format. + +Request +: @@snip [request.sh](assets/multi-fetch/request.sh) + +Payload +: @@snip [payload.json](assets/multi-fetch/payload.json) + +Response +: @@snip [response.json](assets/multi-fetch/response.json) + diff --git a/docs/src/main/paradox/docs/releases/index.md b/docs/src/main/paradox/docs/releases/index.md index a57338979d..11fcf42b50 100644 --- a/docs/src/main/paradox/docs/releases/index.md +++ b/docs/src/main/paradox/docs/releases/index.md @@ -31,8 +31,9 @@ The latest stable release is **v1.8.0** released on **14.06.2023**. ### New features / enhancements - @ref:[Aggregations of resources by `@type` and `project`](../delta/api/resources-api.md#aggregations) -- @ref:[Resources can be added to an archive using `_self`](../delta/api/archives-api.md#payload) +- @ref:[Files can be added to an archive using `_self`](../delta/api/archives-api.md#payload) - @ref:[Indexing errors can now be listed and filtered](../delta/api/views/index.md#listing-indexing-failures) +- @ref:[Multi fetch operation allows to get multiple resources in a single call](../delta/api/multi-fetch.md) ## 1.8.0 (14.06.2023) diff --git a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md index 06ff1196a3..ba21f656db 100644 --- a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md @@ -8,6 +8,12 @@ TODO add potential migration page ### Resources +#### Multi fetch + +Multiple resources can now be retrieved within a single call with the multi-fetch operation. + +@ref:[More information](../delta/api/multi-fetch.md) + #### Payload validation It is now forbidden for JSON payloads to contain fields beginning with underscore (_). This can be disabled be setting `app.resources.decoding-option` to `lenient`, however it is not recommended as specification of this data in payloads can have unexpected consequences in both data and the user-interface diff --git a/tests/src/test/resources/kg/multi-fetch/all-success.json b/tests/src/test/resources/kg/multi-fetch/all-success.json new file mode 100644 index 0000000000..4ad49b30f6 --- /dev/null +++ b/tests/src/test/resources/kg/multi-fetch/all-success.json @@ -0,0 +1,40 @@ +{ + "format" : "source", + "resources" : [ + { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/resource?tag=v1.0.0", + "project" : "{{project1}}", + "value" : { + "@context" : { + "nxv" : "https://bluebrain.github.io/nexus/vocabulary/", + "other" : "https://some.other.prefix.com/" + }, + "@type" : "nxv:TestResource", + "other:priority" : 5, + "other:projects" : [ + "testProject", + "testProject2" + ] + } + }, + { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/file", + "project" : "{{project2}}", + "value" : { + "_bytes" : 47, + "_digest" : { + "_algorithm" : "SHA-256", + "_value" : "00ff4b34e3f3695c3abcdec61cba72c2238ed172ef34ae1196bfad6a4ec23dda" + }, + "_filename" : "attachment.json", + "_mediaType" : "application/json", + "_origin" : "Client", + "_storage" : { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault", + "@type" : "https://bluebrain.github.io/nexus/vocabulary/DiskStorage", + "_rev" : 1 + } + } + } + ] +} \ No newline at end of file diff --git a/tests/src/test/resources/kg/multi-fetch/limited-access.json b/tests/src/test/resources/kg/multi-fetch/limited-access.json new file mode 100644 index 0000000000..db3dc24fcf --- /dev/null +++ b/tests/src/test/resources/kg/multi-fetch/limited-access.json @@ -0,0 +1,32 @@ +{ + "format" : "source", + "resources" : [ + { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/resource?tag=v1.0.0", + "project" : "{{project1}}", + "error" : { + "@type" : "AuthorizationFailed", + "reason" : "The supplied authentication is not authorized to access this resource." + } + }, + { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/file", + "project" : "{{project2}}", + "value" : { + "_bytes" : 47, + "_digest" : { + "_algorithm" : "SHA-256", + "_value" : "00ff4b34e3f3695c3abcdec61cba72c2238ed172ef34ae1196bfad6a4ec23dda" + }, + "_filename" : "attachment.json", + "_mediaType" : "application/json", + "_origin" : "Client", + "_storage" : { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault", + "@type" : "https://bluebrain.github.io/nexus/vocabulary/DiskStorage", + "_rev" : 1 + } + } + } + ] +} \ No newline at end of file diff --git a/tests/src/test/resources/kg/multi-fetch/unknown.json b/tests/src/test/resources/kg/multi-fetch/unknown.json new file mode 100644 index 0000000000..2fa4110f85 --- /dev/null +++ b/tests/src/test/resources/kg/multi-fetch/unknown.json @@ -0,0 +1,13 @@ +{ + "format" : "source", + "resources" : [ + { + "@id" : "https://bluebrain.github.io/nexus/vocabulary/xxx", + "project" : "{{project1}}", + "error": { + "@type": "NotFound", + "reason": "The resource 'https://bluebrain.github.io/nexus/vocabulary/xxx' was not found in project '{{project1}}'." + } + } + ] +} \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index cc3a03699d..5e12991ed9 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -117,6 +117,11 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit as: ActorSyst )(implicit um: FromEntityUnmarshaller[A]): Task[Assertion] = requestAssert(PATCH, url, Some(body), identity, extraHeaders)(assertResponse) + def getWithBody[A](url: String, body: Json, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( + assertResponse: (A, HttpResponse) => Assertion + )(implicit um: FromEntityUnmarshaller[A]): Task[Assertion] = + requestAssert(GET, url, Some(body), identity, extraHeaders)(assertResponse) + def get[A](url: String, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): Task[Assertion] = diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala index 0c8de754bc..08c3a86a42 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Optics.scala @@ -21,11 +21,16 @@ object Optics extends Optics { val filtered = keys.foldLeft(jsonObject) { (o, k) => o.remove(k) } JsonObject.fromIterable( filtered.toList.map { case (k, v) => - v.asObject.fold(k -> v) { o => - k -> Json.fromJsonObject( - inner(o) - ) - } + v.arrayOrObject( + k -> v, + a => + k -> Json.fromValues( + a.map { element => + element.asObject.fold(element) { e => Json.fromJsonObject(inner(e)) } + } + ), + o => k -> Json.fromJsonObject(inner(o)) + ) } ) } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala new file mode 100644 index 0000000000..3756e3796e --- /dev/null +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala @@ -0,0 +1,112 @@ +package ch.epfl.bluebrain.nexus.tests.kg + +import akka.http.scaladsl.model.{ContentTypes, StatusCodes} +import ch.epfl.bluebrain.nexus.tests.BaseSpec +import ch.epfl.bluebrain.nexus.tests.Identity.listings.{Alice, Bob} +import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Organizations, Resources} +import io.circe.Json +import ch.epfl.bluebrain.nexus.tests.Optics._ + +class MultiFetchSpec extends BaseSpec { + + private val org1 = genId() + private val proj11 = genId() + private val proj12 = genId() + private val ref11 = s"$org1/$proj11" + private val ref12 = s"$org1/$proj12" + + private val prefix = "https://bluebrain.github.io/nexus/vocabulary/" + + override def beforeAll(): Unit = { + super.beforeAll() + + val setup = for { + _ <- aclDsl.addPermission("/", Bob, Organizations.Create) + _ <- adminDsl.createOrganization(org1, org1, Bob) + _ <- adminDsl.createProject(org1, proj11, kgDsl.projectJson(name = proj11), Bob) + _ <- adminDsl.createProject(org1, proj12, kgDsl.projectJson(name = proj12), Bob) + _ <- aclDsl.addPermission(s"/$ref12", Alice, Resources.Read) + } yield () + + val resourcePayload = + jsonContentOf( + "/kg/resources/simple-resource.json", + "priority" -> "5" + ) + + val createResources = for { + // Creation + _ <- deltaClient.put[Json](s"/resources/$ref11/_/nxv:resource", resourcePayload, Bob)(expectCreated) + _ <- deltaClient.putAttachment[Json]( + s"/files/$ref12/nxv:file", + contentOf("/kg/files/attachment.json"), + ContentTypes.`application/json`, + "attachment.json", + Bob + )(expectCreated) + // Tag + _ <- deltaClient.post[Json](s"/resources/$ref11/_/nxv:resource/tags?rev=1", tag("v1.0.0", 1), Bob)(expectCreated) + } yield () + + (setup >> createResources).accepted + } + + "Fetching multiple resources" should { + + def request(format: String) = + json""" + { + "format": "$format", + "resources": [ + { "id": "${prefix}resource?tag=v1.0.0", "project": "$ref11" }, + { "id": "${prefix}file", "project": "$ref12" } + ] + }""" + + "get all resources for a user with all access" in { + val expected = jsonContentOf( + "/kg/multi-fetch/all-success.json", + "project1" -> ref11, + "project2" -> ref12 + ) + + deltaClient.getWithBody[Json]("/multi-fetch/resources", request("source"), Bob) { (json, response) => + response.status shouldEqual StatusCodes.OK + filterNestedKeys("_uuid")(json) shouldEqual expected + } + } + + "get all resources for a user with limited access" in { + val expected = jsonContentOf( + "/kg/multi-fetch/limited-access.json", + "project1" -> ref11, + "project2" -> ref12 + ) + + deltaClient.getWithBody[Json]("/multi-fetch/resources", request("source"), Alice) { (json, response) => + response.status shouldEqual StatusCodes.OK + filterNestedKeys("_uuid")(json) shouldEqual expected + } + } + + "get a not found error for an non-existing-resource" in { + val request = + json""" + { + "format": "source", + "resources": [ + { "id": "${prefix}xxx", "project": "$ref11" } + ] + }""" + + val expected = jsonContentOf("/kg/multi-fetch/unknown.json", "project1" -> ref11) + + deltaClient.getWithBody[Json]("/multi-fetch/resources", request, Bob) { (json, response) => + response.status shouldEqual StatusCodes.OK + json shouldEqual expected + } + } + + } + +}