From d0bef8fea4bf7c1e8ba77a90c0592ae58585290a Mon Sep 17 00:00:00 2001 From: Daniel Bell Date: Fri, 5 Jul 2024 10:41:34 +0200 Subject: [PATCH] Patch view resource types (#5055) * Patch resource types in blazegraph views * Patch resource types in ES view values --- .../BlazegraphDecoderConfiguration.scala | 2 +- .../model/BlazegraphViewValue.scala | 10 +- .../model/ElasticSearchViewValue.scala | 2 +- .../ship/views/BlazegraphViewProcessor.scala | 4 +- .../views/ElasticSearchViewProcessor.scala | 4 +- .../nexus/ship/views/ViewPatcher.scala | 48 ++++++- .../nexus/ship/views/ViewPatcherSuite.scala | 120 +++++++++++++++++- 7 files changed, 171 insertions(+), 19 deletions(-) diff --git a/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphDecoderConfiguration.scala b/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphDecoderConfiguration.scala index d5549de91b..26a1f1f2ae 100644 --- a/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphDecoderConfiguration.scala +++ b/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphDecoderConfiguration.scala @@ -6,7 +6,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, JsonLdContext, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.Configuration -private[blazegraph] object BlazegraphDecoderConfiguration { +object BlazegraphDecoderConfiguration { def apply(implicit jsonLdApi: JsonLdApi, rcr: RemoteContextResolution): IO[Configuration] = for { contextValue <- IO.delay { ContextValue(contexts.blazegraph) } diff --git a/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/model/BlazegraphViewValue.scala b/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/model/BlazegraphViewValue.scala index ab521d7039..b200154dfb 100644 --- a/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/model/BlazegraphViewValue.scala +++ b/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/model/BlazegraphViewValue.scala @@ -8,13 +8,15 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.configuration.semiauto.d import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.{Configuration, JsonLdDecoder} import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission import ch.epfl.bluebrain.nexus.delta.sdk.views.ViewRef +import ch.epfl.bluebrain.nexus.delta.sourcing.Serializer import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.{Latest, UserTag} import ch.epfl.bluebrain.nexus.delta.sourcing.model.IriFilter import ch.epfl.bluebrain.nexus.delta.sourcing.query.SelectFilter import ch.epfl.bluebrain.nexus.delta.sourcing.stream.PipeChain -import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +import io.circe.generic.extras +import io.circe.generic.extras.semiauto.{deriveConfiguredCodec, deriveConfiguredEncoder} import io.circe.syntax._ -import io.circe.{Encoder, Json} +import io.circe.{Codec, Encoder, Json} /** * Enumeration of Blazegraph view values. @@ -150,4 +152,8 @@ object BlazegraphViewValue { ): JsonLdDecoder[BlazegraphViewValue] = deriveConfigJsonLdDecoder[BlazegraphViewValue] + object Database { + implicit private val configuration: extras.Configuration = Serializer.circeConfiguration + implicit val bgvvCodec: Codec.AsObject[BlazegraphViewValue] = deriveConfiguredCodec[BlazegraphViewValue] + } } diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/model/ElasticSearchViewValue.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/model/ElasticSearchViewValue.scala index 7eb60fc78b..1bb7231bea 100644 --- a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/model/ElasticSearchViewValue.scala +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/model/ElasticSearchViewValue.scala @@ -213,7 +213,7 @@ object ElasticSearchViewValue { } object Database { - implicit val configuration: Configuration = Serializer.circeConfiguration + implicit private val configuration: Configuration = Serializer.circeConfiguration implicit val valueCodec: Codec.AsObject[ElasticSearchViewValue] = deriveConfiguredCodec[ElasticSearchViewValue] } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala index 45fbf14196..e949481800 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala @@ -48,14 +48,14 @@ class BlazegraphViewProcessor private ( e.id match { case id if id == defaultViewId => IO.unit // the default view is created on project creation case _ => - val patchedSource = viewPatcher.patchAggregateViewSource(e.source) + val patchedSource = viewPatcher.patchBlazegraphViewSource(e.source) views(event.uuid).flatMap(_.create(e.id, project, patchedSource)) } case e: BlazegraphViewUpdated => e.id match { case id if id == defaultViewId => IO.unit case _ => - val patchedSource = viewPatcher.patchAggregateViewSource(e.source) + val patchedSource = viewPatcher.patchBlazegraphViewSource(e.source) views(event.uuid).flatMap(_.update(e.id, project, cRev, patchedSource)) } case e: BlazegraphViewDeprecated => diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala index fc858af56f..c2397fd682 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala @@ -48,14 +48,14 @@ class ElasticSearchViewProcessor private ( e.id match { case id if id == defaultViewId => IO.unit // the default view is created on project creation case _ => - val patchedSource = viewPatcher.patchAggregateViewSource(e.source) + val patchedSource = viewPatcher.patchElasticSearchViewSource(e.source) views(event.uuid).flatMap(_.create(e.id, project, patchedSource)) } case e: ElasticSearchViewUpdated => e.id match { case id if id == defaultViewId => IO.unit case _ => - val patchedSource = viewPatcher.patchAggregateViewSource(e.source) + val patchedSource = viewPatcher.patchElasticSearchViewSource(e.source) views(event.uuid).flatMap(_.update(e.id, project, cRev, patchedSource)) } case e: ElasticSearchViewDeprecated => diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcher.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcher.scala index 76d1817223..44245fb5ca 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcher.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcher.scala @@ -10,7 +10,47 @@ import io.circe.syntax.EncoderOps final class ViewPatcher(projectMapper: ProjectMapper, iriPatcher: IriPatcher) { - def patchAggregateViewSource(input: Json): Json = + private def patchGenericViewResourceTypes(input: Json): Json = + root.resourceTypes.arr.each.modify(patchResourceType)(input) + + private def patchResourceType(json: Json) = + patchIri(json) + .getOrElse( + throw new IllegalArgumentException(s"Invalid resource type found in Blazegraph view resource types: $json") + ) + + private def patchIri(json: Json) = { + json + .as[Iri] + .map { iri => + iriPatcher(iri).asJson + } + } + + def patchBlazegraphViewSource(input: Json): Json = { + patchGenericViewResourceTypes( + patchAggregateViewSource(input) + ) + } + + def patchElasticSearchViewSource(input: Json): Json = { + patchPipelineResourceTypes( + patchGenericViewResourceTypes( + patchAggregateViewSource(input) + ) + ) + } + + private def patchPipelineResourceTypes(input: Json): Json = { + root.pipeline.each.config.each.`https://bluebrain.github.io/nexus/vocabulary/types`.each.`@id`.string + .modify(patchStringIri)(input) + } + + private def patchStringIri(stringIri: String): String = { + Iri.apply(stringIri).map(iriPatcher.apply).map(_.toString).getOrElse(stringIri) + } + + private def patchAggregateViewSource(input: Json): Json = root.views.each.obj.modify { view => view .mapAllKeys("project", patchProject) @@ -26,11 +66,7 @@ final class ViewPatcher(projectMapper: ProjectMapper, iriPatcher: IriPatcher) { .getOrElse(throw new IllegalArgumentException(s"Invalid project ref found in aggregate view source: $json")) private def patchViewId(json: Json) = - json - .as[Iri] - .map { iri => - iriPatcher(iri).asJson - } + patchIri(json) .getOrElse(throw new IllegalArgumentException(s"Invalid view id found in aggregate view source: $json")) } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcherSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcherSuite.scala index 278d7225e4..c2299b95ba 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcherSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/views/ViewPatcherSuite.scala @@ -1,18 +1,68 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.data.NonEmptySet -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewValue -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewValue.AggregateElasticSearchViewValue +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewValue +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewValue.Database._ +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewValue.IndexingBlazegraphViewValue +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.ElasticSearchViewJsonLdSourceDecoder +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{contexts, ElasticSearchViewValue} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewValue.{AggregateElasticSearchViewValue, IndexingElasticSearchViewValue} import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewValue.Database._ -import ch.epfl.bluebrain.nexus.delta.sdk.views.ViewRef -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.contexts.{elasticsearch, elasticsearchMetadata} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.schemas +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.views.{PipeStep, ViewRef} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{IriFilter, ProjectRef} +import ch.epfl.bluebrain.nexus.delta.sourcing.stream.pipes.FilterByType import ch.epfl.bluebrain.nexus.ship.{IriPatcher, ProjectMapper} import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import io.circe.syntax.EncoderOps +import java.util.UUID + class ViewPatcherSuite extends NexusSuite { + implicit val rcr: RemoteContextResolution = RemoteContextResolution.fixedIOResource( + elasticsearch -> ContextValue.fromFile("contexts/elasticsearch.json"), + elasticsearchMetadata -> ContextValue.fromFile("contexts/elasticsearch-metadata.json"), + contexts.aggregations -> ContextValue.fromFile("contexts/aggregations.json"), + contexts.elasticsearchIndexing -> ContextValue.fromFile("contexts/elasticsearch-indexing.json"), + Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), + Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json"), + Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), + Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json"), + Vocabulary.contexts.shacl -> ContextValue.fromFile("contexts/shacl.json"), + Vocabulary.contexts.statistics -> ContextValue.fromFile("contexts/statistics.json"), + Vocabulary.contexts.offset -> ContextValue.fromFile("contexts/offset.json"), + Vocabulary.contexts.pipeline -> ContextValue.fromFile("contexts/pipeline.json"), + Vocabulary.contexts.tags -> ContextValue.fromFile("contexts/tags.json"), + Vocabulary.contexts.search -> ContextValue.fromFile("contexts/search.json") + ) + implicit val api: JsonLdApi = JsonLdJavaApi.strict + private val ref = ProjectRef.unsafe("org", "proj") + private val context = ProjectContext.unsafe( + ApiMappings("_" -> schemas.resources, "resource" -> schemas.resources), + iri"http://localhost/v1/resources/org/proj/_/", + iri"http://schema.org/", + enforceSchema = false + ) + + implicit private val uuidF: UUIDF = UUIDF.fixed(UUID.randomUUID()) + + implicit private val resolverContext: ResolverContextResolution = ResolverContextResolution(rcr) + + implicit private val caller: Caller = Caller.Anonymous + private val decoder = + ElasticSearchViewJsonLdSourceDecoder(uuidF, resolverContext).unsafeRunSync() + private val project1 = ProjectRef.unsafe("org", "project") private val viewId1 = iri"https://bbp.epfl.ch/view1" private val view1 = ViewRef(project1, viewId1) @@ -33,13 +83,73 @@ class ViewPatcherSuite extends NexusSuite { private val projectMapper = ProjectMapper(Map(project2 -> targetProject)) private val viewPatcher = new ViewPatcher(projectMapper, iriPatcher) + private def indexingESViewWithFilterByTypePipeline(types: Iri*): IndexingElasticSearchViewValue = { + IndexingElasticSearchViewValue( + None, + None, + pipeline = filterByTypePipeline(types: _*) + ) + } + + private def filterByTypePipeline(types: Iri*) = List( + PipeStep(FilterByType(IriFilter.fromSet(types.toSet))) + ) + test("Patch the aggregate view") { val viewAsJson = aggregateView.asJson val expectedView1 = ViewRef(project1, iri"https://openbrainplatform.com/view1") val expectedView2 = ViewRef(targetProject, viewId2) val expectedAggregated = AggregateElasticSearchViewValue(None, None, NonEmptySet.of(expectedView1, expectedView2)) - val result = viewPatcher.patchAggregateViewSource(viewAsJson).as[ElasticSearchViewValue] + val result = viewPatcher.patchElasticSearchViewSource(viewAsJson).as[ElasticSearchViewValue] assertEquals(result, Right(expectedAggregated)) } + test("Patch an ES view's resource types") { + val view = indexingESViewWithFilterByTypePipeline(originalPrefix / "Type1", originalPrefix / "Type2") + val viewAsJson = view.asInstanceOf[ElasticSearchViewValue].asJson + val expectedView = indexingESViewWithFilterByTypePipeline(targetPrefix / "Type1", targetPrefix / "Type2") + val result = viewPatcher.patchElasticSearchViewSource(viewAsJson).as[ElasticSearchViewValue] + assertEquals(result, Right(expectedView)) + } + + test("Patch a legacy ES view's resource types") { + val sourceJson = json"""{ + "@type": "ElasticSearchView", + "resourceTypes": [ "${originalPrefix / "Type1"}", "${originalPrefix / "Type2"}" ], + "mapping": { } + }""" + + val (_, originalValue) = decoder(ref, context, sourceJson).accepted + assertEquals( + originalValue.asIndexingValue.map(_.pipeline.filter(_.name.value == "filterByType")), + Some(filterByTypePipeline(originalPrefix / "Type1", originalPrefix / "Type2")) + ) + + val patchedJson = viewPatcher.patchElasticSearchViewSource(sourceJson) + val (_, patchedValue) = decoder(ref, context, patchedJson).accepted + assertEquals( + patchedValue.asIndexingValue.map(_.pipeline.filter(_.name.value == "filterByType")), + Some(filterByTypePipeline(targetPrefix / "Type1", targetPrefix / "Type2")) + ) + } + + test("Patch a blazegraph view's resource types") { + val view: BlazegraphViewValue = IndexingBlazegraphViewValue(resourceTypes = + IriFilter.fromSet(Set(iri"https://bbp.epfl.ch/resource1", iri"https://bbp.epfl.ch/resource2")) + ) + + val patchedView = viewPatcher.patchBlazegraphViewSource(view.asJson).as[BlazegraphViewValue] + + assertEquals( + patchedView, + Right( + IndexingBlazegraphViewValue(resourceTypes = + IriFilter.fromSet( + Set(iri"https://openbrainplatform.com/resource1", iri"https://openbrainplatform.com/resource2") + ) + ) + ) + ) + } + }