Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Jena and Topbraid versions #4837

Merged
merged 7 commits into from
Apr 5, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ val fs2AwsVersion = "6.1.1"
val googleAuthClientVersion = "1.35.0"
val handleBarsVersion = "4.4.0"
val hikariVersion = "5.1.0"
val jenaVersion = "4.2.0"
val jenaVersion = "4.10.0"
val jsonldjavaVersion = "0.13.6"
val kamonVersion = "2.7.1"
val kanelaAgentVersion = "1.0.18"
Expand All @@ -57,7 +57,7 @@ val postgresJdbcVersion = "42.7.3"
val pureconfigVersion = "0.17.6"
val scalaTestVersion = "3.2.18"
val scalaXmlVersion = "2.2.0"
val topBraidVersion = "1.3.2" // 1.4.1 fails to validate some test schemas
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you encounter any particular issues with validation for some schemas?

Copy link
Contributor Author

@imsdu imsdu Apr 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only when loading the extensions from topbraid where they added more constraints in 1.4.x and some were breaking.
As we don't rely on them, I made the change to only rely on those from shacl core.
There are still a few tests to be done though (via the validate endpoints) to make sure things are ok

val topBraidVersion = "1.4.3" // 1.4.1 fails to validate some test schemas
val testContainersVersion = "1.19.7"
val testContainersScalaVersion = "0.41.3"

Expand Down Expand Up @@ -109,7 +109,7 @@ lazy val fs2Aws = "io.laserdisc" %% "fs2-
lazy val fs2AwsS3 = "io.laserdisc" %% "fs2-aws-s3" % fs2AwsVersion
lazy val googleAuthClient = "com.google.oauth-client" % "google-oauth-client" % googleAuthClientVersion
lazy val handleBars = "com.github.jknack" % "handlebars" % handleBarsVersion
lazy val jenaArq = "org.apache.jena" % "jena-arq" % jenaVersion
lazy val jenaArq = "org.apache.jena" % "jena-arq" % jenaVersion exclude ("com.apicatalog", "titanium-json-ld")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To make sure we don't use Json-LD 1.1 for now

lazy val jsonldjava = "com.github.jsonld-java" % "jsonld-java" % jsonldjavaVersion
lazy val kamonAkkaHttp = "io.kamon" %% "kamon-akka-http" % kamonVersion
lazy val kamonCore = "io.kamon" %% "kamon-core" % kamonVersion
Expand All @@ -127,7 +127,7 @@ lazy val pureconfigCats = "com.github.pureconfig" %% "pure
lazy val scalaReflect = "org.scala-lang" % "scala-reflect" % scalaCompilerVersion
lazy val scalaTest = "org.scalatest" %% "scalatest" % scalaTestVersion
lazy val scalaXml = "org.scala-lang.modules" %% "scala-xml" % scalaXmlVersion
lazy val topBraidShacl = "org.topbraid" % "shacl" % topBraidVersion
lazy val topBraidShacl = "org.topbraid" % "shacl" % topBraidVersion exclude ("com.apicatalog", "titanium-json-ld")
lazy val testContainers = "org.testcontainers" % "testcontainers" % testContainersVersion
lazy val testContainersScala = "com.dimafeng" %% "testcontainers-scala-munit" % testContainersScalaVersion
lazy val testContainersScalaLocalStack = "com.dimafeng" %% "testcontainers-scala-localstack-v2" % testContainersScalaVersion
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import ch.epfl.bluebrain.nexus.delta.config.AppConfig
import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl
import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
import ch.epfl.bluebrain.nexus.delta.routes.ResourcesRoutes
import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction.AggregateIndexingAction
Expand Down Expand Up @@ -36,9 +37,8 @@ object ResourcesModule extends ModuleDef {
ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false)
}

make[ValidateResource].from {
(resourceResolution: ResourceResolution[Schema], rcr: RemoteContextResolution @Id("aggregate")) =>
ValidateResource(resourceResolution)(rcr)
make[ValidateResource].from { (resourceResolution: ResourceResolution[Schema], validateShacl: ValidateShacl) =>
ValidateResource(resourceResolution, validateShacl)
}

make[ResourcesConfig].from { (config: AppConfig) => config.resources }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUID
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.contexts
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ShaclShapesGraph
import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl
import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
import ch.epfl.bluebrain.nexus.delta.routes.SchemasRoutes
import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction.AggregateIndexingAction
Expand Down Expand Up @@ -36,10 +36,9 @@ object SchemasModule extends ModuleDef {

implicit private val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass)

make[ValidateSchema].fromEffect { (api: JsonLdApi, rcr: RemoteContextResolution @Id("aggregate")) =>
ShaclShapesGraph.shaclShaclShapes.map(ValidateSchema(api, _, rcr))
make[ValidateShacl].fromEffect { (rcr: RemoteContextResolution @Id("aggregate")) => ValidateShacl(rcr) }

}
make[ValidateSchema].from { (validateShacl: ValidateShacl, api: JsonLdApi) => ValidateSchema(validateShacl)(api) }

make[SchemaDefinition].from { (validateSchema: ValidateSchema, clock: Clock[IO]) =>
Schemas.definition(validateSchema, clock)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@
},
"value": "nxv:42"
},
"targetedNodes": 417
"targetedNodes": 412
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this changing?

Copy link
Contributor Author

@imsdu imsdu Apr 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not totally sure but we don't load the dash/tosh extensions provided by Topbraid anymore so my guess is there are fewer constraints and fewer nodes can get validated in the end.
targetedNodes is also a Nexus thing and if a node validated by several constraints will be accounted several times.

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils}
import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv, schema, schemas}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl
import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction
import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
Expand Down Expand Up @@ -96,7 +97,8 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
}

private val validator: ValidateResource = ValidateResource(
ResourceResolutionGen.singleInProject(projectRef, fetchSchema)
ResourceResolutionGen.singleInProject(projectRef, fetchSchema),
ValidateShacl(rcr).accepted
)
private val fetchContext = FetchContextDummy(List(project.value))
private val resolverContextResolution: ResolverContextResolution = ResolverContextResolution(rcr)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ShaclShapesGraph
import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl
import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction
import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
Expand All @@ -23,7 +23,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.schemas
import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy
import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, SchemasConfig, SchemasImpl, ValidateSchema}
import ch.epfl.bluebrain.nexus.delta.sdk.schemas._
import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec
import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog
import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User}
Expand All @@ -37,9 +37,8 @@ import java.util.UUID

class SchemasRoutesSpec extends BaseRouteSpec with IOFromMap with CatsIOValues {

private val uuid = UUID.randomUUID()
implicit private val uuidF: UUIDF = UUIDF.fixed(uuid)
implicit private val shaclShaclShapes: ShaclShapesGraph = ShaclShapesGraph.shaclShaclShapes.accepted
private val uuid = UUID.randomUUID()
implicit private val uuidF: UUIDF = UUIDF.fixed(uuid)

private val reader = User("reader", realm)
private val writer = User("writer", realm)
Expand Down Expand Up @@ -79,7 +78,7 @@ class SchemasRoutesSpec extends BaseRouteSpec with IOFromMap with CatsIOValues {

private val config = SchemasConfig(eventLogConfig)

private val schemaDef = Schemas.definition(ValidateSchema.apply, clock)
private val schemaDef = Schemas.definition(ValidateSchema(ValidateShacl(rcr).accepted), clock)
private lazy val schemaLog = ScopedEventLog(schemaDef, config.eventLog, xas)

private lazy val routes =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import org.apache.jena.sparql.graph.GraphFactory
import org.apache.jena.sparql.util.IsoMatcher

import java.util.UUID
import scala.annotation.tailrec
import scala.annotation.{nowarn, tailrec}
import scala.jdk.CollectionConverters._
import scala.util.Try

Expand All @@ -39,6 +39,7 @@ import scala.util.Try
* @param value
* the Jena dataset graph
*/
@nowarn("cat=deprecation")
final case class Graph private (rootNode: IriOrBNode, value: DatasetGraph) { self =>

val rootResource: Node = subject(rootNode)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ import org.apache.jena.riot.system.ErrorHandlerFactory
import org.apache.jena.riot._
import org.apache.jena.sparql.core.DatasetGraph

import scala.annotation.nowarn
import scala.jdk.CollectionConverters._
import scala.util.Try

/**
* Json-LD high level API implementation by Json-LD Java library
*/
@nowarn("cat=deprecation")
final class JsonLdJavaApi(config: JsonLdApiConfig) extends JsonLdApi {

System.setProperty(DocumentLoader.DISALLOW_REMOTE_CONTEXT_LOADING, "true")
Expand Down Expand Up @@ -76,7 +78,7 @@ final class JsonLdJavaApi(config: JsonLdApiConfig) extends JsonLdApi {
val ds = DatasetFactory.create
val initBuilder = RDFParser.create
.fromString(input.noSpaces)
.lang(Lang.JSONLD)
.lang(Lang.JSONLD10)
.context(c)
.strict(config.strict)
.checking(config.extraChecks)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
package ch.epfl.bluebrain.nexus.delta.rdf.shacl

import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxsh
import ch.epfl.bluebrain.nexus.delta.rdf.graph.Graph
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
import org.apache.jena.query.{Dataset, DatasetFactory}
import org.apache.jena.query.Dataset
import org.apache.jena.rdf.model._
import org.topbraid.jenax.util.JenaDatatypes
import org.topbraid.shacl.engine.{Constraint, ShapesGraph}
import org.topbraid.shacl.validation.{ValidationEngine, ValidationEngineConfiguration}
import org.topbraid.shacl.validation.ValidationEngine

import java.net.URI
import java.util
Expand All @@ -26,7 +23,7 @@ import java.util
* the ShapesGraph with the shapes to validate against
*/
@SuppressWarnings(Array("NullParameter"))
final class ShaclEngine private (dataset: Dataset, shapesGraphURI: URI, shapesGraph: ShapesGraph)
final class ShaclEngine(dataset: Dataset, shapesGraphURI: URI, shapesGraph: ShapesGraph)
extends ValidationEngine(dataset, shapesGraphURI, shapesGraph, null) {
private var targetedNodes = 0

Expand All @@ -43,82 +40,3 @@ final class ShaclEngine private (dataset: Dataset, shapesGraphURI: URI, shapesGr
private def toProperty(iri: Iri): Property =
ResourceFactory.createProperty(iri.toString)
}

object ShaclEngine {

/**
* Validates a given graph against the SHACL shapes spec.
*
* @param shapesGraph
* the shapes Graph to test against the SHACL shapes spec
* @param reportDetails
* true to also include the sh:detail (more verbose) and false to omit them
* @return
* an option of [[ValidationReport]] with the validation results
*/
def apply(
shapesGraph: Graph,
reportDetails: Boolean
)(implicit shaclShapesGraph: ShaclShapesGraph, rcr: RemoteContextResolution): IO[ValidationReport] =
apply(shapesGraph, shaclShapesGraph, validateShapes = true, reportDetails = reportDetails)

/**
* Validates a given data Graph against all shapes from a given shapes Model.
*
* @param dataGraph
* the data Graph
* @param shapesGraph
* the shapes Graph
* @param reportDetails
* true to also include the sh:detail (more verbose) and false to omit them
* @return
* an option of [[ValidationReport]] with the validation results
*/
def apply(
dataGraph: Graph,
shapesGraph: Graph,
reportDetails: Boolean
)(implicit rcr: RemoteContextResolution): IO[ValidationReport] =
apply(dataGraph, ShaclShapesGraph(shapesGraph), validateShapes = false, reportDetails)

/**
* Validates a given data Graph against all shapes from a given shapes graph.
*
* @param graph
* the data Graph
* @param shapesGraph
* the shapes graph
* @param validateShapes
* true to also validate the shapes graph
* @param reportDetails
* true to also include the sh:detail (more verbose) and false to omit them
* @return
* an option of [[ValidationReport]] with the validation results
*/
def apply(
graph: Graph,
shapesGraph: ShaclShapesGraph,
validateShapes: Boolean,
reportDetails: Boolean
)(implicit rcr: RemoteContextResolution): IO[ValidationReport] =
apply(DatasetFactory.wrap(graph.value), shapesGraph, validateShapes, reportDetails)

private def apply(
dataset: Dataset,
shapesGraph: ShaclShapesGraph,
validateShapes: Boolean,
reportDetails: Boolean
)(implicit rcr: RemoteContextResolution): IO[ValidationReport] = {
// Create Dataset that contains both the data model and the shapes model
// (here, using a temporary URI for the shapes graph)
dataset.addNamedModel(shapesGraph.uri.toString, shapesGraph.model)
val engine = new ShaclEngine(dataset, shapesGraph.uri, shapesGraph.value)
engine.setConfiguration(
new ValidationEngineConfiguration().setReportDetails(reportDetails).setValidateShapes(validateShapes)
)
IO.delay {
engine.applyEntailments()
engine.validateAll()
}.flatMap(ValidationReport(_))
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package ch.epfl.bluebrain.nexus.delta.rdf.shacl

import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader
import org.apache.jena.graph.GraphMemFactory.createDefaultGraph
import org.apache.jena.rdf.model.{Model, ModelFactory}
import org.apache.jena.util.FileUtils
import org.topbraid.shacl.vocabulary.SH

object ShaclFileLoader {

private val loader = ClasspathResourceLoader()

private def readFromTurtleFile(base: String, resourcePath: String) =
loader
.streamOf(resourcePath)
.use { is =>
IO.blocking {
ModelFactory
.createModelForGraph(createDefaultGraph())
.read(is, base, FileUtils.langTurtle)
}
}

def readShaclShapes: IO[Model] = readFromTurtleFile("http://www.w3.org/ns/shacl-shacl#", "shacl-shacl.ttl")

def readShaclVocabulary: IO[Model] = readFromTurtleFile(SH.BASE_URI, "rdf/shacl.ttl")
}

This file was deleted.

Loading
Loading