Allow to add exceptions to schema enforcement (#5075)

Co-authored-by: Simon Dumas <simon.dumas@epfl.ch>
BlueBrain · Aug 6, 2024 · 416863e · 416863e
1 parent 288e893
commit 416863e
Show file tree

Hide file tree

Showing 22 changed files with 403 additions and 357 deletions.
diff --git a/delta/app/src/main/resources/app.conf b/delta/app/src/main/resources/app.conf
@@ -269,6 +269,11 @@ app {
     event-log = ${app.defaults.event-log}
     # Reject payloads which contain nexus metadata fields (any field beginning with _)
     decoding-option = "strict"
+    # Defines exceptions for schema enforcement
+    schema-enforcement {
+      type-whitelist = []
+      allow-no-types = false
+    }
     # Do not create a new revision of a resource when the update does not introduce a change
     skip-update-no-change = true
   }

diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala
@@ -37,8 +37,10 @@ object ResourcesModule extends ModuleDef {
     ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false)
   }
 
-  make[ValidateResource].from { (resourceResolution: ResourceResolution[Schema], validateShacl: ValidateShacl) =>
-    ValidateResource(resourceResolution, validateShacl)
+  make[ValidateResource].from {
+    (resourceResolution: ResourceResolution[Schema], validateShacl: ValidateShacl, config: ResourcesConfig) =>
+      val schemaClaimResolver = SchemaClaimResolver(resourceResolution, config.schemaEnforcement)
+      ValidateResource(schemaClaimResolver, validateShacl)
   }
 
   make[ResourcesConfig].from { (config: AppConfig) => config.resources }

diff --git a/delta/app/src/test/resources/resources/errors/invalid-resource.json b/delta/app/src/test/resources/resources/errors/invalid-resource.json
diff --git a/delta/app/src/test/resources/schemas/errors/invalid-schema-2.json b/delta/app/src/test/resources/schemas/errors/invalid-schema-2.json
@@ -7,19 +7,6 @@
   "reason": "Schema 'https://bluebrain.github.io/nexus/vocabulary/pretendschema' could not be resolved in 'myorg/myproject'",
   "report": {
     "history": [
-      {
-        "rejections": [
-          {
-            "cause": {
-              "@type": "ResourceNotFound",
-              "reason": "The resource was not found in project 'myorg/myproject'."
-            },
-            "project": "myorg/myproject"
-          }
-        ],
-        "resolverId": "https://bluebrain.github.io/nexus/vocabulary/in-project",
-        "success": false
-      }
     ]
   }
 }
diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala
@@ -4,41 +4,37 @@ import akka.http.scaladsl.model.MediaTypes.`text/html`
 import akka.http.scaladsl.model.headers.{Accept, Location, OAuth2BearerToken, RawHeader}
 import akka.http.scaladsl.model.{RequestEntity, StatusCodes, Uri}
 import akka.http.scaladsl.server.Route
-import cats.effect.IO
 import cats.implicits._
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils}
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv, schema => schemaOrg, schemas}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
-import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl
 import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
-import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ProjectGen, ResourceResolutionGen, SchemaGen}
+import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
-import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegmentRef, ResourceUris}
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
 import ch.epfl.bluebrain.nexus.delta.sdk.resources._
-import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema
 import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec
 import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User}
+import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag
-import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
 import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues
 import io.circe.{Json, Printer}
 import org.scalatest.Assertion
 
 import java.util.UUID
 
-class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
+class ResourcesRoutesSpec extends BaseRouteSpec with ValidateResourceFixture with CatsIOValues {
 
   private val uuid                  = UUID.randomUUID()
   implicit private val uuidF: UUIDF = UUIDF.fixed(uuid)
@@ -54,9 +50,9 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
   private val asReader = addCredentials(OAuth2BearerToken("reader"))
   private val asWriter = addCredentials(OAuth2BearerToken("writer"))
 
-  private val am           = ApiMappings("nxv" -> nxv.base, "Person" -> schemaOrg.Person)
-  private val projBase     = nxv.base
-  private val project      = ProjectGen.resourceFor(
+  private val am         = ApiMappings("nxv" -> nxv.base, "Person" -> schemaOrg.Person)
+  private val projBase   = nxv.base
+  private val project    = ProjectGen.resourceFor(
     ProjectGen.project(
       "myorg",
       "myproject",
@@ -66,12 +62,10 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
       mappings = am + Resources.mappings
     )
   )
-  private val projectRef   = project.value.ref
-  private val schemaSource = jsonContentOf("resources/schema.json").addContext(contexts.shacl, contexts.schemasMetadata)
-  private val schema1      = SchemaGen.schema(nxv + "myschema", project.value.ref, schemaSource.removeKeys(keywords.id))
-  private val schema2      = SchemaGen.schema(schemaOrg.Person, project.value.ref, schemaSource.removeKeys(keywords.id))
-  private val schema3      = SchemaGen.schema(nxv + "otherSchema", project.value.ref, schemaSource.removeKeys(keywords.id))
-  private val tag          = UserTag.unsafe("mytag")
+  private val projectRef = project.value.ref
+  private val schema1    = nxv + "myschema"
+  private val schema2    = nxv + "otherSchema"
+  private val tag        = UserTag.unsafe("mytag")
 
   private val myId                            = nxv + "myid" // Resource created against no schema with id present on the payload
   private def encodeWithBase(id: String)      = UrlUtils.encode((nxv + id).toString)
@@ -89,27 +83,13 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
 
   private val aclCheck = AclSimpleCheck().accepted
 
-  private val fetchSchema: (ResourceRef, ProjectRef) => FetchF[Schema] = {
-    case (ref, _) if ref.iri == schema2.id => IO.pure(Some(SchemaGen.resourceFor(schema2, deprecated = true)))
-    case (ref, _) if ref.iri == schema1.id => IO.pure(Some(SchemaGen.resourceFor(schema1)))
-    case (ref, _) if ref.iri == schema3.id => IO.pure(Some(SchemaGen.resourceFor(schema3)))
-    case _                                 => IO.none
-  }
-
-  private val validator: ValidateResource                          = ValidateResource(
-    ResourceResolutionGen.singleInProject(projectRef, fetchSchema),
-    ValidateShacl(rcr).accepted
-  )
+  private val validateResource                                     = validateFor(Set((projectRef, schema1), (projectRef, schema2)))
   private val fetchContext                                         = FetchContextDummy(List(project.value))
   private val resolverContextResolution: ResolverContextResolution = ResolverContextResolution(rcr)
 
   private def routesWithDecodingOption(implicit decodingOption: DecodingOption): (Route, Resources) = {
-    val resourceDef = Resources.definition(validator, DetectChange(enabled = true), clock)
-    val scopedLog   = ScopedEventLog(
-      resourceDef,
-      ResourcesConfig(eventLogConfig, decodingOption, skipUpdateNoChange = true).eventLog,
-      xas
-    )
+    val resourceDef = Resources.definition(validateResource, DetectChange(enabled = true), clock)
+    val scopedLog   = ScopedEventLog(resourceDef, eventLogConfig, xas)
 
     val resources = ResourcesImpl(
       scopedLog,
@@ -158,7 +138,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
     "create a resource" in {
       val endpoints = List(
         ("/v1/resources/myorg/myproject", schemas.resources),
-        ("/v1/resources/myorg/myproject/myschema", schema1.id)
+        ("/v1/resources/myorg/myproject/myschema", schema1)
       )
       forAll(endpoints) { case (endpoint, schema) =>
         val id = genString()
@@ -172,7 +152,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
     "create a tagged resource" in {
       val endpoints           = List(
         ("/v1/resources/myorg/myproject?tag=mytag", schemas.resources),
-        ("/v1/resources/myorg/myproject/myschema?tag=mytag", schema1.id)
+        ("/v1/resources/myorg/myproject/myschema?tag=mytag", schema1)
       )
       val (routes, resources) = routesWithDecodingOption(DecodingOption.Strict)
       forAll(endpoints) { case (endpoint, schema) =>
@@ -188,7 +168,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
     "create a resource with an authenticated user and provided id" in {
       val endpoints = List(
         ((id: String) => s"/v1/resources/myorg/myproject/_/$id", schemas.resources),
-        ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id", schema1.id)
+        ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id", schema1)
       )
       forAll(endpoints) { case (endpoint, schema) =>
         val id = genString()
@@ -202,7 +182,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
     "create a tagged resource with an authenticated user and provided id" in {
       val endpoints           = List(
         ((id: String) => s"/v1/resources/myorg/myproject/_/$id?tag=mytag", schemas.resources),
-        ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id?tag=mytag", schema1.id)
+        ((id: String) => s"/v1/resources/myorg/myproject/myschema/$id?tag=mytag", schema1)
       )
       val (routes, resources) = routesWithDecodingOption(DecodingOption.Strict)
       forAll(endpoints) { case (endpoint, schema) =>
@@ -228,17 +208,6 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
       }
     }
 
-    "fail to create a resource that does not validate against a schema" in {
-      val payloadFailingSchemaConstraints = payloadWithoutId.replaceKeyWithValue("number", "wrong")
-      Put(
-        "/v1/resources/myorg/myproject/nxv:myschema/wrong",
-        payloadFailingSchemaConstraints.toEntity
-      ) ~> asWriter ~> routes ~> check {
-        response.status shouldEqual StatusCodes.BadRequest
-        response.asJson shouldEqual jsonContentOf("resources/errors/invalid-resource.json")
-      }
-    }
-
     "fail to create a resource against a schema that does not exist" in {
       Put(
         "/v1/resources/myorg/myproject/pretendschema/someid",
@@ -296,7 +265,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
         forAll(endpoints) { case (endpoint, rev) =>
           Put(s"$endpoint?rev=$rev", payloadUpdated(id).toEntity(Printer.noSpaces)) ~> asWriter ~> routes ~> check {
             status shouldEqual StatusCodes.OK
-            response.asJson shouldEqual standardWriterMetadata(id, rev = rev + 1, schema1.id)
+            response.asJson shouldEqual standardWriterMetadata(id, rev = rev + 1, schema1)
           }
         }
       }
@@ -341,7 +310,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
         forAll(endpoints) { endpoint =>
           Put(s"$endpoint", payloadUpdated.toEntity(Printer.noSpaces)) ~> asWriter ~> routes ~> check {
             status shouldEqual StatusCodes.OK
-            response.asJson shouldEqual standardWriterMetadata(id, rev = 1, schema = schema1.id)
+            response.asJson shouldEqual standardWriterMetadata(id, rev = 1, schema = schema1)
           }
         }
       }
@@ -384,7 +353,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
       givenAResourceWithSchema("myschema") { id =>
         Put(s"/v1/resources/$projectRef/otherSchema/$id/update-schema") ~> asWriter ~> routes ~> check {
           response.status shouldEqual StatusCodes.OK
-          response.asJson.hcursor.get[String]("_constrainedBy").toOption should contain(schema3.id.toString)
+          response.asJson.hcursor.get[String]("_constrainedBy").toOption should contain(schema2.toString)
         }
       }
     }
@@ -509,7 +478,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
           s"/v1/resources/myorg/myproject/_/$id?rev=1",
           s"/v1/resources/myorg/myproject/$mySchema/$id?tag=$myTag"
         )
-        val meta      = standardWriterMetadata(id, schema = schema1.id, tpe = "schema:Custom")
+        val meta      = standardWriterMetadata(id, schema = schema1, tpe = "schema:Custom")
         forAll(endpoints) { endpoint =>
           Get(endpoint) ~> asReader ~> routes ~> check {
             status shouldEqual StatusCodes.OK

diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesConfig.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesConfig.scala
@@ -1,6 +1,8 @@
 package ch.epfl.bluebrain.nexus.delta.sdk.resources
 
+import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.ResourcesConfig.SchemaEnforcementConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import pureconfig.ConfigReader
 import pureconfig.generic.semiauto.deriveReader
@@ -12,12 +14,31 @@ import pureconfig.generic.semiauto.deriveReader
   *   configuration of the event log
   * @param decodingOption
   *   strict/lenient decoding of resources
+  * @param schemaEnforcement
+  *   configuration related to schema enforcement
   * @param skipUpdateNoChange
   *   do not create a new revision when the update does not introduce a change in the current resource state
   */
-final case class ResourcesConfig(eventLog: EventLogConfig, decodingOption: DecodingOption, skipUpdateNoChange: Boolean)
+final case class ResourcesConfig(
+    eventLog: EventLogConfig,
+    decodingOption: DecodingOption,
+    schemaEnforcement: SchemaEnforcementConfig,
+    skipUpdateNoChange: Boolean
+)
 
 object ResourcesConfig {
-  implicit final val resourcesConfigReader: ConfigReader[ResourcesConfig] =
+
+  /**
+    * Configuration to allow to bypass schema enforcing in some cases
+    * @param typeWhitelist
+    *   types for which a schema is not required
+    * @param allowNoTypes
+    *   allow to skip schema validation for resources without any types
+    */
+  final case class SchemaEnforcementConfig(typeWhitelist: Set[Iri], allowNoTypes: Boolean)
+
+  implicit final val resourcesConfigReader: ConfigReader[ResourcesConfig] = {
+    implicit val schemaEnforcementReader: ConfigReader[SchemaEnforcementConfig] = deriveReader[SchemaEnforcementConfig]
     deriveReader[ResourcesConfig]
+  }
 }
diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/SchemaClaim.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/SchemaClaim.scala
@@ -3,9 +3,6 @@ package ch.epfl.bluebrain.nexus.delta.sdk.resources
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.schemas
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.SchemaClaim._
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.ValidationResult._
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.SchemaIsMandatory
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
 
 /**
@@ -16,32 +13,8 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
   */
 sealed trait SchemaClaim {
 
-  /**
-    * Validate the claim
-    * @param enforceSchema
-    *   to ban unconstrained resources
-    * @param submitOnDefinedSchema
-    *   the function to call when a schema is defined
-    */
-  def validate(enforceSchema: Boolean)(submitOnDefinedSchema: SubmitOnDefinedSchema): IO[ValidationResult] =
-    this match {
-      case CreateWithSchema(project, schema, caller) =>
-        submitOnDefinedSchema(project, schema, caller)
-      case CreateUnconstrained(project)              =>
-        onUnconstrained(project, enforceSchema)
-      case UpdateToSchema(project, schema, caller)   =>
-        submitOnDefinedSchema(project, schema, caller)
-      case UpdateToUnconstrained(project)            =>
-        onUnconstrained(project, enforceSchema)
-      case KeepUnconstrained(project)                =>
-        IO.pure(NoValidation(project))
-    }
-
   def project: ProjectRef
 
-  private def onUnconstrained(project: ProjectRef, enforceSchema: Boolean) =
-    IO.raiseWhen(enforceSchema)(SchemaIsMandatory(project)).as(NoValidation(project))
-
 }
 
 object SchemaClaim {
@@ -52,16 +25,16 @@ object SchemaClaim {
     def schemaRef: ResourceRef
   }
 
-  final private case class CreateWithSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
+  final case class CreateWithSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
       extends DefinedSchemaClaim
-  final private case class CreateUnconstrained(project: ProjectRef) extends SchemaClaim
+  final case class CreateUnconstrained(project: ProjectRef) extends SchemaClaim
 
-  final private case class UpdateToSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
+  final case class UpdateToSchema(project: ProjectRef, schemaRef: ResourceRef, caller: Caller)
       extends DefinedSchemaClaim
 
-  final private case class UpdateToUnconstrained(project: ProjectRef) extends SchemaClaim
+  final case class UpdateToUnconstrained(project: ProjectRef) extends SchemaClaim
 
-  final private case class KeepUnconstrained(project: ProjectRef) extends SchemaClaim
+  final case class KeepUnconstrained(project: ProjectRef) extends SchemaClaim
 
   private def isUnconstrained(schema: ResourceRef): Boolean = schema.iri == schemas.resources