Skip to content

Commit

Permalink
Merge branch 'master' into fix-listing-flaky-tests
Browse files Browse the repository at this point in the history
  • Loading branch information
imsdu authored Aug 21, 2023
2 parents b2e8c41 + b71d5a8 commit 8a1ea27
Show file tree
Hide file tree
Showing 123 changed files with 3,814 additions and 1,480 deletions.
13 changes: 11 additions & 2 deletions .github/workflows/ci-integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,18 @@ jobs:
app/Docker/publishLocal \
storage/Docker/publishLocal
- name: Start services
run: docker-compose -f tests/docker/docker-compose.yml up -d && sleep 60
run: docker-compose -f tests/docker/docker-compose.yml up -d && sleep 70
- name: Test
run: sbt -Dsbt.color=always -Dsbt.supershell=false "project tests" test
run: |
URL="http://localhost:8080/v1/version"
RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
if [ $RESPONSE_CODE -eq 200 ]; then
sbt -Dsbt.color=always -Dsbt.supershell=false "project tests" test
else
RED='\033[0;31m'; NC='\033[0m'
echo "${RED}Aborting:${NC} The Delta /v1/version returned HTTP $RESPONSE_CODE."
exit 1
fi
- name: Stop & clean Docker
if: ${{ always() }}
run: docker-compose -f tests/docker/docker-compose.yml down --rmi "local" --volumes
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class PostgresServiceDependencySpec extends DoobieScalaTestFixture with Matchers

"fetch its service name and version" in {
new PostgresServiceDependency(xas).serviceDescription.accepted shouldEqual
ServiceDescription(Name.unsafe("postgres"), "15.3")
ServiceDescription(Name.unsafe("postgres"), "15.4")
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
"@container": "@set"
},
"sources": {
"@container": "@set"
"@container": "@list"
},
"projections": {
"@container": "@set"
"@container": "@list"
},
"_uuid": "https://bluebrain.github.io/nexus/vocabulary/uuid"
},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews

import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.BlazegraphViews
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.SparqlQueryResponseType.Aux
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client._
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.projectionNamespace
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.CompositeViewDef.ActiveViewDef
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.indexing.{commonNamespace, projectionNamespace}
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewProjection.SparqlProjection
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewRejection.{AuthorizationFailed, ViewIsDeprecated, WrappedBlazegraphClientError}
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{CompositeView, CompositeViewRejection, ViewResource, ViewSparqlProjectionResource}
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewRejection
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewRejection.{AuthorizationFailed, WrappedBlazegraphClientError}
import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery
import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef}
import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment
import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
import monix.bio.IO

Expand Down Expand Up @@ -82,12 +82,6 @@ trait BlazegraphQuery {
}

object BlazegraphQuery {

private[compositeviews] type FetchView =
(IdSegmentRef, ProjectRef) => IO[CompositeViewRejection, ViewResource]
private[compositeviews] type FetchProjection =
(IdSegment, IdSegment, ProjectRef) => IO[CompositeViewRejection, ViewSparqlProjectionResource]

final def apply(
aclCheck: AclCheck,
views: CompositeViews,
Expand All @@ -96,16 +90,16 @@ object BlazegraphQuery {
): BlazegraphQuery =
BlazegraphQuery(
aclCheck,
views.fetch,
views.fetchBlazegraphProjection,
views.fetchIndexingView,
views.expand,
client,
prefix
)

private[compositeviews] def apply(
aclCheck: AclCheck,
fetchView: FetchView,
fetchProjection: FetchProjection,
expandId: ExpandId,
client: SparqlQueryClient,
prefix: String
): BlazegraphQuery =
Expand All @@ -118,11 +112,10 @@ object BlazegraphQuery {
responseType: Aux[R]
)(implicit caller: Caller): IO[CompositeViewRejection, R] =
for {
viewRes <- fetchView(id, project)
_ <- IO.raiseWhen(viewRes.deprecated)(ViewIsDeprecated(viewRes.id))
permissions = viewRes.value.projections.map(_.permission)
_ <- aclCheck.authorizeForEveryOr(project, permissions.toSortedSet)(AuthorizationFailed)
namespace = BlazegraphViews.namespace(viewRes.value.uuid, viewRes.rev, prefix)
view <- fetchView(id, project)
permissions = view.sparqlProjections.map(_.permission)
_ <- aclCheck.authorizeForEveryOr(project, permissions)(AuthorizationFailed)
namespace = commonNamespace(view.uuid, view.indexingRev, prefix)
result <- client.query(Set(namespace), query, responseType).mapError(WrappedBlazegraphClientError)
} yield result

Expand All @@ -134,12 +127,11 @@ object BlazegraphQuery {
responseType: Aux[R]
)(implicit caller: Caller): IO[CompositeViewRejection, R] =
for {
viewRes <- fetchProjection(id, projectionId, project)
_ <- IO.raiseWhen(viewRes.deprecated)(ViewIsDeprecated(viewRes.id))
(view, projection) = viewRes.value
_ <- aclCheck.authorizeForOr(project, projection.permission)(AuthorizationFailed)
namespace = projectionNamespace(projection, view.uuid, viewRes.rev, prefix)
result <- client.query(Set(namespace), query, responseType).mapError(WrappedBlazegraphClientError)
view <- fetchView(id, project)
projection <- fetchProjection(view, projectionId)
_ <- aclCheck.authorizeForOr(project, projection.permission)(AuthorizationFailed)
namespace = projectionNamespace(projection, view.uuid, prefix)
result <- client.query(Set(namespace), query, responseType).mapError(WrappedBlazegraphClientError)
} yield result

override def queryProjections[R <: SparqlQueryResponse](
Expand All @@ -149,24 +141,25 @@ object BlazegraphQuery {
responseType: Aux[R]
)(implicit caller: Caller): IO[CompositeViewRejection, R] =
for {
viewRes <- fetchView(id, project)
_ <- IO.raiseWhen(viewRes.deprecated)(ViewIsDeprecated(viewRes.id))
view = viewRes.value
namespaces <- allowedProjections(view, viewRes.rev, project)
view <- fetchView(id, project)
namespaces <- allowedProjections(view, project)
result <- client.query(namespaces, query, responseType).mapError(WrappedBlazegraphClientError)
} yield result

private def allowedProjections(
view: CompositeView,
rev: Int,
project: ProjectRef
)(implicit caller: Caller): IO[AuthorizationFailed, Set[String]] =
private def fetchProjection(view: ActiveViewDef, projectionId: IdSegment) =
expandId(projectionId, view.project).flatMap { id =>
IO.fromEither(view.sparqlProjection(id))
}

private def allowedProjections(view: ActiveViewDef, project: ProjectRef)(implicit
caller: Caller
): IO[AuthorizationFailed, Set[String]] =
aclCheck
.mapFilterAtAddress[SparqlProjection, String](
view.projections.value.collect { case p: SparqlProjection => p },
view.sparqlProjections,
project,
p => p.permission,
p => projectionNamespace(p, view.uuid, rev, prefix)
p => projectionNamespace(p, view.uuid, prefix)
)
.tapEval { namespaces => IO.raiseWhen(namespaces.isEmpty)(AuthorizationFailed) }
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews

import cats.data.NonEmptyList
import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeView.RebuildStrategy
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{CompositeViewFields, CompositeViewProjection, CompositeViewProjectionFields, CompositeViewSource, CompositeViewSourceFields, CompositeViewValue}
import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectBase
import ch.epfl.bluebrain.nexus.delta.sdk.views.IndexingRev
import monix.bio.UIO

object CompositeViewFactory {

/**
* Create a new [[CompositeViewValue]] from [[CompositeViewFields]].
*
* - The indexing revision for the common spaces and the projections defaults to 1.
*/
def create(fields: CompositeViewFields)(implicit projectBase: ProjectBase, uuidF: UUIDF): UIO[CompositeViewValue] =
for {
sources <- fields.sources.traverse { create }
projections <- fields.projections.traverse { create(_, IndexingRev.init) }
} yield CompositeViewValue(
fields.name,
fields.description,
IndexingRev.init,
sources.toNem,
projections.toNem,
fields.rebuildStrategy
)

/**
* Update a [[CompositeViewValue]] from [[CompositeViewFields]] and the previous [[CompositeViewValue]].
*
* - When a source or a projection keep the same id, the existing uuid is also preserved
* - When any source is added/updated/deleted, the indexing revision for the common space and all the projections
* are updated
* - When a projection is updated, only the indexing revision of this projection is updated
*/
def update(fields: CompositeViewFields, current: CompositeViewValue, nextRev: IndexingRev)(implicit
projectBase: ProjectBase,
uuidF: UUIDF
): UIO[CompositeViewValue] = {
for {
sources <- fields.sources.traverse { upsert(_, current.sources.lookup) }.map(_.toNem)
// If any source has changed, we update the indexing rev for sources
sourceHasChanged = current.sources != sources
newSourceIndexingRev = if (sourceHasChanged) nextRev else current.sourceIndexingRev
projections <- fields.projections
.traverse {
upsert(_, current.projections.lookup, nextRev, sourceHasChanged)
}
.map(_.toNem)
} yield CompositeViewValue(
fields.name,
fields.description,
newSourceIndexingRev,
sources,
projections,
fields.rebuildStrategy
)
}

// Generate an id and a uuid for a source or a projection
private def generate(implicit projectBase: ProjectBase, uuidF: UUIDF) = uuidF().map { uuid =>
uuid -> projectBase.iri / uuid.toString
}

private[compositeviews] def create(
input: CompositeViewSourceFields
)(implicit projectBase: ProjectBase, uuidF: UUIDF) =
generate.map { case (uuid, id) =>
val source = input.toSource(uuid, id)
source.id -> source
}

// Create or update a source, preserving the existing uuid if it exists
private[compositeviews] def upsert(
input: CompositeViewSourceFields,
find: Iri => Option[CompositeViewSource]
)(implicit projectBase: ProjectBase, uuidF: UUIDF) = {
val currentSourceOpt = input.id.flatMap(find)
currentSourceOpt
.map { currentSource =>
UIO.pure {
currentSource.id -> input.toSource(currentSource.uuid, currentSource.id)
}
}
.getOrElse {
create(input)
}
}

private[compositeviews] def create(input: CompositeViewProjectionFields, nextRev: IndexingRev)(implicit
projectBase: ProjectBase,
uuidF: UUIDF
) =
generate.map { case (uuid, id) =>
val projection = input.toProjection(uuid, id, nextRev)
projection.id -> projection
}

// Create or update a projection, preserving the existing uuid if it exists
private[compositeviews] def upsert(
input: CompositeViewProjectionFields,
find: Iri => Option[CompositeViewProjection],
newRev: IndexingRev,
sourceHasChanged: Boolean
)(implicit projectBase: ProjectBase, uuidF: UUIDF) = {
val currentProjectionOpt = input.id.flatMap(find)
currentProjectionOpt
.map { currentProjection =>
UIO.pure {
val newProjection =
input.toProjection(currentProjection.uuid, currentProjection.id, currentProjection.indexingRev)
val newIndexingRev =
if (sourceHasChanged || currentProjection != newProjection) newRev else currentProjection.indexingRev
currentProjection.id -> newProjection.updateIndexingRev(newIndexingRev)
}
}
.getOrElse {
create(input, newRev)
}
}

/**
* Construct a [[CompositeViewValue]] without name and description
*
* Meant for testing purposes
*/
def unsafe(
sources: NonEmptyList[CompositeViewSource],
projections: NonEmptyList[CompositeViewProjection],
rebuildStrategy: Option[RebuildStrategy]
): CompositeViewValue =
CompositeViewValue(
None,
None,
IndexingRev(1),
sources.map { s => s.id -> s }.toNem,
projections.map { p => p.id -> p }.toNem,
rebuildStrategy
)
}
Loading

0 comments on commit 8a1ea27

Please sign in to comment.