diff --git a/build.sbt b/build.sbt index be8257bc85..f055d63232 100755 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ val akkaHttpCirceVersion = "1.39.2" val akkaCorsVersion = "1.2.0" val akkaVersion = "2.6.21" val alpakkaVersion = "3.0.4" -val awsSdkVersion = "2.29.2" +val awsSdkVersion = "2.29.9" val betterMonadicForVersion = "0.3.1" val caffeineVersion = "3.1.8" val catsEffectVersion = "3.5.5" @@ -37,7 +37,7 @@ val fs2Version = "3.11.0" val fs2AwsVersion = "6.1.3" val googleAuthClientVersion = "1.36.0" val handleBarsVersion = "4.4.0" -val hikariVersion = "6.0.0" +val hikariVersion = "6.1.0" val jenaVersion = "4.10.0" val jsonldjavaVersion = "0.13.6" val kamonVersion = "2.7.5" @@ -48,7 +48,7 @@ val logbackVersion = "1.5.12" val magnoliaVersion = "1.1.10" val munitVersion = "1.0.2" val munitCatsEffectVersion = "2.0.0" -val nimbusJoseJwtVersion = "9.42" +val nimbusJoseJwtVersion = "9.46" val postgresJdbcVersion = "42.7.4" val pureconfigVersion = "0.17.7" val scalaTestVersion = "3.2.19" @@ -713,6 +713,7 @@ lazy val ship = project compositeViewsPlugin % "compile->compile", elasticsearchPlugin % "compile->compile", storagePlugin % "compile->compile;test->test", + searchPlugin, tests % "test->compile;test->test" ) .settings( diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf index f820b1b66c..2438c16ab3 100644 --- a/delta/plugins/storage/src/main/resources/storage.conf +++ b/delta/plugins/storage/src/main/resources/storage.conf @@ -44,7 +44,8 @@ plugins.storage { show-location = true # the default maximum allowed file size (in bytes) for uploaded files. 10 GB default-max-file-size = 10737418240 - default-bucket = "nexus-delta-production" + default-bucket = "default-bucket-override-me" + default-bucket = ${?DEFAULT_BUCKET} } # Remote disk storage configuration remote-disk { diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index cd0efca7e7..b816cd3761 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -467,7 +467,7 @@ Request (metadata) Response (metadata) : @@snip [fetched-metadata.json](assets/files/fetched-metadata.json) -If the @ref:[redirect to Fusion feature](../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch tags diff --git a/docs/src/main/paradox/docs/delta/api/projects-api.md b/docs/src/main/paradox/docs/delta/api/projects-api.md index ebcfe34501..b718a79e25 100644 --- a/docs/src/main/paradox/docs/delta/api/projects-api.md +++ b/docs/src/main/paradox/docs/delta/api/projects-api.md @@ -253,7 +253,7 @@ Request Response : @@snip [fetched.json](assets/projects/fetched.json) -If the @ref:[redirect to Fusion feature](../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the project will be returned. ## Fetch (specific version) diff --git a/docs/src/main/paradox/docs/delta/api/resolvers-api.md b/docs/src/main/paradox/docs/delta/api/resolvers-api.md index 0acb3f229f..8cb8204032 100644 --- a/docs/src/main/paradox/docs/delta/api/resolvers-api.md +++ b/docs/src/main/paradox/docs/delta/api/resolvers-api.md @@ -210,7 +210,7 @@ Request Response : @@snip [fetched.json](assets/resolvers/fetched.json) -If the @ref:[redirect to Fusion feature](../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch original payload diff --git a/docs/src/main/paradox/docs/delta/api/resources-api.md b/docs/src/main/paradox/docs/delta/api/resources-api.md index edd4653e14..e48b2425fd 100644 --- a/docs/src/main/paradox/docs/delta/api/resources-api.md +++ b/docs/src/main/paradox/docs/delta/api/resources-api.md @@ -327,7 +327,7 @@ Request Response : @@snip [fetched.json](assets/resources/fetched.json) -If the @ref:[redirect to Fusion feature](../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch original payload diff --git a/docs/src/main/paradox/docs/delta/api/schemas-api.md b/docs/src/main/paradox/docs/delta/api/schemas-api.md index b7cefb4844..6b5a8c18bb 100644 --- a/docs/src/main/paradox/docs/delta/api/schemas-api.md +++ b/docs/src/main/paradox/docs/delta/api/schemas-api.md @@ -247,7 +247,7 @@ Request Response : @@snip [schema-fetched.json](assets/schemas/fetched.json) -If the @ref:[redirect to Fusion feature](../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch original payload diff --git a/docs/src/main/paradox/docs/delta/api/search-api.md b/docs/src/main/paradox/docs/delta/api/search-api.md index 4b89be9f41..0fa2bdb26f 100644 --- a/docs/src/main/paradox/docs/delta/api/search-api.md +++ b/docs/src/main/paradox/docs/delta/api/search-api.md @@ -15,7 +15,7 @@ The search plugin relies on @ref:[composite views](./views/composite-view-api.md indices to which user has access to, i.e. has `views/query` permission. For instructions on how to configure global search in Nexus and how it works please visit the -@ref:[Search configuration](../../getting-started/running-nexus/search-configuration.md) page. +@ref:[Search configuration](../../running-nexus/search-configuration.md) page. @@@ note { .tip title="Api Mapping" } diff --git a/docs/src/main/paradox/docs/delta/api/storages-api.md b/docs/src/main/paradox/docs/delta/api/storages-api.md index d760280508..3232e02266 100644 --- a/docs/src/main/paradox/docs/delta/api/storages-api.md +++ b/docs/src/main/paradox/docs/delta/api/storages-api.md @@ -221,7 +221,7 @@ Request Response : @@snip [fetched.json](assets/storages/fetched.json) -If the @ref:[redirect to Fusion feature](../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch original payload diff --git a/docs/src/main/paradox/docs/delta/api/supervision-api.md b/docs/src/main/paradox/docs/delta/api/supervision-api.md index 4e4888be1a..819f7d0ad6 100644 --- a/docs/src/main/paradox/docs/delta/api/supervision-api.md +++ b/docs/src/main/paradox/docs/delta/api/supervision-api.md @@ -64,7 +64,7 @@ and the number of triples per individual view. The unassigned part allows to spot orphan namespaces, that is to say namespaces which may not have been properly deleted when a view got modified or deprecated. -This is an indicator which helps to scale correctly the @ref:[Blazegraph instance](../../getting-started/running-nexus/blazegraph.md). +This is an indicator which helps to scale correctly the @ref:[Blazegraph instance](../../running-nexus/blazegraph.md). ``` GET /v1/supervision/blazegraph diff --git a/docs/src/main/paradox/docs/delta/api/views/composite-sink.md b/docs/src/main/paradox/docs/delta/api/views/composite-sink.md index 6ec7fe89f5..c719041418 100644 --- a/docs/src/main/paradox/docs/delta/api/views/composite-sink.md +++ b/docs/src/main/paradox/docs/delta/api/views/composite-sink.md @@ -16,7 +16,7 @@ selected via configuration. By default, Nexus Delta will use the Single Composite Sink. This sink performs one query to the Blazegraph common namespace for each resource in the project. The queries are done in chronological order (by the `updatedAt` time of the resources). -We recommend reading through the @ref:[search configuration example use case](../../../getting-started/running-nexus/search-configuration.md#example-use-case) and the @ref[Composite View API reference](composite-view-api.md) to learn more about Composite Views. +We recommend reading through the @ref:[search configuration example use case](../../../running-nexus/search-configuration.md#example-use-case) and the @ref[Composite View API reference](composite-view-api.md) to learn more about Composite Views. ## Batch Composite Sink diff --git a/docs/src/main/paradox/docs/delta/api/views/composite-view-api.md b/docs/src/main/paradox/docs/delta/api/views/composite-view-api.md index 8b3c34ccd4..7af1e406cd 100644 --- a/docs/src/main/paradox/docs/delta/api/views/composite-view-api.md +++ b/docs/src/main/paradox/docs/delta/api/views/composite-view-api.md @@ -500,7 +500,7 @@ Request Response : @@snip [fetched.json](../assets/views/composite/fetched.json) -If the @ref:[redirect to Fusion feature](../../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch original payload diff --git a/docs/src/main/paradox/docs/delta/api/views/elasticsearch-view-api.md b/docs/src/main/paradox/docs/delta/api/views/elasticsearch-view-api.md index 10a3f126d9..1936f963ef 100644 --- a/docs/src/main/paradox/docs/delta/api/views/elasticsearch-view-api.md +++ b/docs/src/main/paradox/docs/delta/api/views/elasticsearch-view-api.md @@ -332,7 +332,7 @@ Request Response : @@snip [fetched.json](../assets/views/elasticsearch/fetched.json) -If the @ref:[redirect to Fusion feature](../../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. Note that for retro-compatibility purposes, fetching an elasticsearch view returns legacy fields. diff --git a/docs/src/main/paradox/docs/delta/api/views/sparql-view-api.md b/docs/src/main/paradox/docs/delta/api/views/sparql-view-api.md index 33414becb4..f216cb80b7 100644 --- a/docs/src/main/paradox/docs/delta/api/views/sparql-view-api.md +++ b/docs/src/main/paradox/docs/delta/api/views/sparql-view-api.md @@ -211,7 +211,7 @@ Request Response : @@snip [fetched.json](../assets/views/blazegraph/sparql/fetched.json) -If the @ref:[redirect to Fusion feature](../../../getting-started/running-nexus/configuration/index.md#fusion-configuration) is enabled and +If the @ref:[redirect to Fusion feature](../../../running-nexus/configuration/index.md#fusion-configuration) is enabled and if the `Accept` header is set to `text/html`, a redirection to the fusion representation of the resource will be returned. ## Fetch original payload diff --git a/docs/src/main/paradox/docs/delta/plugins/index.md b/docs/src/main/paradox/docs/delta/plugins/index.md index 78ae8a65c1..d7c4ec902b 100644 --- a/docs/src/main/paradox/docs/delta/plugins/index.md +++ b/docs/src/main/paradox/docs/delta/plugins/index.md @@ -77,7 +77,7 @@ The plugin can also define instances of following traits/classes, which will be - @link:[EventMetricEncoder](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetricEncoder.scala){ open=new } - enables Delta to convert a database event to an event metric - @link:[MetadataContextValue](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/MetadataContextValue.scala){ open=new } - registers metadata context of this plugin into global metadata context - @link:[RemoteContextResolution](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/context/RemoteContextResolution.scala){ open=new } - enables Delta to resolve static contexts defined by the plugin - - @link:[ServiceDependency](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/ServiceDependency.scala){ open=new } - allows the plugin to define dependencies which will be displayed in `/version` endpoint. + - @link:[ServiceDependency](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/dependency/ServiceDependency.scala){ open=new } - allows the plugin to define dependencies which will be displayed in `/version` endpoint. - @link:[ApiMappings](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/model/ApiMappings.scala){ open=new } - allows the plugin to define default API mappings used to shorten URLs - @link:[ResourceToSchemaMappings](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceToSchemaMappings.scala){ open=new } - allows the plugin to define mapping from the resource type to schema, which can be used to interact with resources created by the plugin through `/resources` endpoints. diff --git a/docs/src/main/paradox/docs/faq.md b/docs/src/main/paradox/docs/faq.md index 756a14120b..6b0c16a47f 100644 --- a/docs/src/main/paradox/docs/faq.md +++ b/docs/src/main/paradox/docs/faq.md @@ -17,8 +17,8 @@ There are many ways to run Nexus. Our public Sandbox is running @link:[here](htt and you can use it to test Nexus on small, non-sensitive data. Our @ref:[tutorial](getting-started/try-nexus.md) can help you to run Nexus step by step. -Meanwhile if you want to run it locally you can do so using @ref:[Docker](getting-started/running-nexus/index.md#docker). You can also deploy Nexus -@ref:[“on premise”](getting-started/running-nexus/index.md#on-premise-cloud-deployment), as a single instance or as a cluster. +Meanwhile if you want to run it locally you can do so using @ref:[Docker](running-nexus/index.md#docker). You can also deploy Nexus +@ref:[“on premise”](running-nexus/index.md#on-premise-cloud-deployment), as a single instance or as a cluster. Blue Brain Nexus has also been deployed and tested on AWS using @link:[Kubernetes](https://kubernetes.io/){ open=new }. ### How can I try Blue Brain Nexus without installing it? @@ -75,7 +75,7 @@ There are several channels provided to address different issues: On macOS and Windows, Docker effectively runs containers inside a VM created by the system hypervisor. Nexus requires at least 2 CPUs and 8 GB of memory in total. You can increase the limits in Docker settings in the menu -_Preferences > Advanced_. More details are in the dedicated @ref:[page](getting-started/running-nexus/index.md). +_Preferences > Advanced_. More details are in the dedicated @ref:[page](running-nexus/index.md). ### What is JSON-LD? diff --git a/docs/src/main/paradox/docs/getting-started/index.md b/docs/src/main/paradox/docs/getting-started/index.md index ed1cd3c16f..84ff7b3ff4 100644 --- a/docs/src/main/paradox/docs/getting-started/index.md +++ b/docs/src/main/paradox/docs/getting-started/index.md @@ -3,7 +3,6 @@ - @ref:[Understanding Knowledge Graphs](understanding-knowledge-graphs.md) - @ref:[Neuroscience Tutorial](try-nexus.md) - @ref:[MovieLens Tutorial](try-nexus-movielens.md) -- @ref:[Running Nexus](running-nexus/index.md) @@@ @@ -13,4 +12,4 @@ The Nexus ecosystem is a data catalog with several technologies and techniques u One of the powerful ways to query data is through the Knowledge Graph built in Nexus. For that reason we encourage you to get familiar with these concepts reading @ref:[Understanding Knowledge Graphs](understanding-knowledge-graphs.md). -The next steps would be to either try Nexus using the @ref:[Sandbox](try-nexus.md) deployment or @ref:[run Nexus locally](running-nexus/index.md). \ No newline at end of file +The next steps would be to either try Nexus using the @ref:[Sandbox](try-nexus.md) deployment or @ref:[run Nexus locally](../running-nexus/index.md). \ No newline at end of file diff --git a/docs/src/main/paradox/docs/getting-started/try-nexus.md b/docs/src/main/paradox/docs/getting-started/try-nexus.md index 66c7804c36..18ebbc79f7 100644 --- a/docs/src/main/paradox/docs/getting-started/try-nexus.md +++ b/docs/src/main/paradox/docs/getting-started/try-nexus.md @@ -116,7 +116,7 @@ You will need the names of the organisation and project for the next parts of th ### 1.2. Running your Own Instance of Nexus -We do not recommend to setup your own instance for this tutorial. If you are interested to install your own instance, @ref:[check our guide](./running-nexus/index.md). +We do not recommend to setup your own instance for this tutorial. If you are interested to install your own instance, @ref:[check our guide](../running-nexus/index.md). ## Step 2: Download and Register Data from a Single Source diff --git a/docs/src/main/paradox/docs/index.md b/docs/src/main/paradox/docs/index.md index efd6ad618c..7a2df45594 100644 --- a/docs/src/main/paradox/docs/index.md +++ b/docs/src/main/paradox/docs/index.md @@ -2,8 +2,8 @@ - [Versions](versions.md) - [Quick Start Guide](getting-started/index.md) +- [Running Nexus](running-nexus/index.md) - [Releases](releases/index.md) -- [Roadmap](roadmap.md) - [Nexus Fusion](fusion/index.md) - [Nexus Forge](forge.md) - [Nexus Delta](delta/index.md) @@ -67,14 +67,10 @@ Do you know know what a knowledge graph is? Or want to deepen your understanding Want to get started with Nexus right away? We provide a @ref:[tutorial](getting-started/try-nexus.md) that uses our @link:[sandbox](https://sandbox.bluebrainnexus.io/){ open=new } environment to let you try the functionalities. -Ready to deploy your own instance of Nexus? Follow our @ref:[guide](getting-started/running-nexus/index.md). +Ready to deploy your own instance of Nexus? Follow our @ref:[guide](running-nexus/index.md). ## Versions Looking for docs for another release of Nexus ? @ref:[List of versions](versions.md) - -## Roadmap - -Curious about our upcoming features? Read our @ref:[open Roadmap](roadmap.md) to find out more. diff --git a/docs/src/main/paradox/docs/releases/v1.11-release-notes.md b/docs/src/main/paradox/docs/releases/v1.11-release-notes.md index c52969ed30..52c0444679 100644 --- a/docs/src/main/paradox/docs/releases/v1.11-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.11-release-notes.md @@ -14,19 +14,25 @@ For the detailed list of updates in this release, see the @link:[list of address This feature allows to reduce the pressure on the primary store by changing the way indexing routines for views (but also graph-analytics) behaves when they have processed all available data. -More @ref:[here](../delta/api/views/index.md#passivation) +@ref:[More here](../delta/api/views/index.md#passivation) ### Conditional requests -Nexus now supports conditional requests +Nexus now supports conditional requests. -More @ref[here](../delta/api/conditional-requests.md) +@ref[More here](../delta/api/conditional-requests.md) + +### Realm provisioning + +Nexus now allows to provision realms at start up so as to simplify automated deployments. + +@ref[More here](../running-nexus/configuration/index.md#realm-provisioning) ### Remote storage server The remote storage server part has been removed. -The client is deprecated and remains with @link:[the remote storage in 1.10](https://hub.docker.com/r/bluebrain/nexus-storage/tags). +The client is deprecated and remains compatible with @link:[the remote storage in 1.10](https://hub.docker.com/r/bluebrain/nexus-storage/tags). ## Nexus Fusion diff --git a/docs/src/main/paradox/docs/releases/v1.6-release-notes.md b/docs/src/main/paradox/docs/releases/v1.6-release-notes.md index c3242ef80f..d84af33459 100644 --- a/docs/src/main/paradox/docs/releases/v1.6-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.6-release-notes.md @@ -12,7 +12,7 @@ visit the @ref:[migration instructions](v1.5-to-v1.6-migration.md) for the neces ## Nexus Delta -The main goal of this release is to allow users to search across multiple projects through @ref:[global search](../getting-started/running-nexus/search-configuration.md) +The main goal of this release is to allow users to search across multiple projects through @ref:[global search](../running-nexus/search-configuration.md) Secondary goals include: @@ -47,14 +47,14 @@ Global search is composed of 2 parts: - A Delta plugin which allows to index and query resources according to a given configuration - A matching plugin in Fusion allows to query the endpoints provided by Delta to filter/sort results and customize how these results are displayed to the users -More details about search and how to configure it is provided @ref:[here](../getting-started/running-nexus/search-configuration.md) +More details about search and how to configure it is provided @ref:[here](../running-nexus/search-configuration.md) Please note that this first version is experimental in both Delta and Fusion and that improvements and possibly breaking changes may be introduced in the upcoming releases. ### Secure the Delta Elasticsearch client -With 1.6, it is now possible to define credentials to query a secured Elasticsearch cluster: @ref:[Elasticsearch configuration](../getting-started/running-nexus/configuration/index.md#elasticsearch-views-plugin-configuration). +With 1.6, it is now possible to define credentials to query a secured Elasticsearch cluster: @ref:[Elasticsearch configuration](../running-nexus/configuration/index.md#elasticsearch-views-plugin-configuration). ### Dedicated Blazegraph client for user-defined queries @@ -82,7 +82,7 @@ The RDF parser allowing to validate incoming data can now be @link:[configured]( When enabled, a dedicated project is created for the current user on its first access to the Nexus platform. -How to enable and configure it is detailed @ref:[here](../getting-started/running-nexus/configuration/index.md#automatic-project-provisioning) +How to enable and configure it is detailed @ref:[here](../running-nexus/configuration/index.md#automatic-project-provisioning) ### Deletion of projects diff --git a/docs/src/main/paradox/docs/releases/v1.7-release-notes.md b/docs/src/main/paradox/docs/releases/v1.7-release-notes.md index e34f0f4a39..1d756149ad 100644 --- a/docs/src/main/paradox/docs/releases/v1.7-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.7-release-notes.md @@ -90,7 +90,7 @@ This change implies a @ref:[migration when updating Nexus Delta to 1.7](v1.6-to- Besides JVM properties and environment variables, Delta configuration can now be defined via a external file. -@ref:[Read More about Delta configuration](../getting-started/running-nexus/configuration/index.md) +@ref:[Read More about Delta configuration](../running-nexus/configuration/index.md) ### Deleting tags on resources diff --git a/docs/src/main/paradox/docs/releases/v1.7-to-v1.8-migration.md b/docs/src/main/paradox/docs/releases/v1.7-to-v1.8-migration.md index 4f07005b6d..89bc2af22b 100644 --- a/docs/src/main/paradox/docs/releases/v1.7-to-v1.8-migration.md +++ b/docs/src/main/paradox/docs/releases/v1.7-to-v1.8-migration.md @@ -248,7 +248,7 @@ The following environment variables need to be set when launching Delta v1.8 in | S3 storage enabling | plugins.storage.storages.amazon.enabled | true/false | For more information on the configuration of Delta, please refer to the -@ref:[Nexus configuration page](../getting-started/running-nexus/configuration/index.md). +@ref:[Nexus configuration page](../running-nexus/configuration/index.md). ## Migrating events from Cassandra to PostgreSQL diff --git a/docs/src/main/paradox/docs/releases/v1.8-to-v1.9-migration.md b/docs/src/main/paradox/docs/releases/v1.8-to-v1.9-migration.md index 8ab2185f7d..c0d4388eb4 100644 --- a/docs/src/main/paradox/docs/releases/v1.8-to-v1.9-migration.md +++ b/docs/src/main/paradox/docs/releases/v1.8-to-v1.9-migration.md @@ -61,5 +61,5 @@ Currently, both remote storages and composite views with remote sources can hold A remote storage might currently have a token stored in the storage definition. In 1.9, this should be moved to `plugins.storage.storages.remote-disk.credentials`. -A composite view might currently have a token stored in the view definition. In 1.9, this should be moved to `plugins.composite-views.remote-source-credentials`, as shown @ref:[here](../getting-started/running-nexus/configuration/index.md#composite-views-plugin-configuration) +A composite view might currently have a token stored in the view definition. In 1.9, this should be moved to `plugins.composite-views.remote-source-credentials`, as shown @ref:[here](../running-nexus/configuration/index.md#composite-views-plugin-configuration) diff --git a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md index 6b19de5b03..6d0acc327d 100644 --- a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md @@ -151,7 +151,7 @@ Preserving the intermediate space also reduces the fragmentation of the Blazegra Rather than storing an auth token in the composite view, it is now expected to specify an authentication method in the Delta configuration. -@ref:[More information](../getting-started/running-nexus/configuration/index.md#composite-views-plugin-configuration) +@ref:[More information](../running-nexus/configuration/index.md#composite-views-plugin-configuration) #### Remove the resourceTag field in projections of composite views @@ -224,7 +224,7 @@ The automatic detection of the media type can now be customized at the Delta lev NB: The media type provided by the user still prevails over automatic detection. -@ref:[More information](../getting-started/running-nexus/configuration/index.md#file-configuration) +@ref:[More information](../running-nexus/configuration/index.md#file-configuration) #### Tagging at creation/update @@ -307,7 +307,7 @@ The 1.9 release of Nexus Fusion focuses on enhancing user interaction, resource The Nexus storage app also benefits from the changes in automatic file detection. -@ref:[More information](../getting-started/running-nexus/configuration/index.md#file-configuration) +@ref:[More information](../running-nexus/configuration/index.md#file-configuration) ### Removing Delta callback to validate the caller identity diff --git a/docs/src/main/paradox/docs/roadmap.md b/docs/src/main/paradox/docs/roadmap.md deleted file mode 100644 index f56dabeb19..0000000000 --- a/docs/src/main/paradox/docs/roadmap.md +++ /dev/null @@ -1,12 +0,0 @@ -# Roadmap - -Check our @link:[GitHub Milestones](https://github.com/BlueBrain/nexus/milestones?direction=asc&sort=title&state=open){ open=new } for the latest updates and priorities for Nexus. - -Note that the content of milestones is subject to change without notice (for example depending on sprint completion). -We favor frequent releases, so we might remove some items of the milestone instead of extending the due date. - -Our current release schedule is one milestone every two weeks approximately, resulting in the next patch, minor or major release. - -The schedule might vary depending on urgency of patch releases, feature complexity, and time of year (among other things). - -Please reach out to us on our @link:[GitHub Discussions](https://github.com/BlueBrain/nexus/discussions){ open=new } page if you have questions. diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/blazegraph.md b/docs/src/main/paradox/docs/running-nexus/blazegraph.md similarity index 94% rename from docs/src/main/paradox/docs/getting-started/running-nexus/blazegraph.md rename to docs/src/main/paradox/docs/running-nexus/blazegraph.md index aac68d7994..2b1ec1320d 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/blazegraph.md +++ b/docs/src/main/paradox/docs/running-nexus/blazegraph.md @@ -38,8 +38,8 @@ It is likely to happen when reindexing after updating search but if it happens r Blazegraph will use the available RAM in 2 ways, JVM heap and the file system cache so like Elasticsearch, the JVM garbage collection frequency and duration are also important to monitor. -Using the supervision endpoints for @ref:[Blazegraph](../../delta/api/supervision-api.md#blazegraph) -and @ref:[composite views](../../delta/api/supervision-api.md#composite-views) and +Using the supervision endpoints for @ref:[Blazegraph](../delta/api/supervision-api.md#blazegraph) +and @ref:[composite views](../delta/api/supervision-api.md#composite-views) and the @link:[RAM guidance from Blazegraph](https://github.com/blazegraph/database/wiki/Hardware_Configuration#ram-sizing-guidance) gives some insights on how much memory can be assigned. @@ -56,8 +56,8 @@ to reduce the disk usage, but it takes quite a bit a time and requires taking th An alternative can be to reindex on a fresh instance of Blazegraph, this approach also allows to reconfigure the underlying namespaces. -Using the supervision endpoints for @ref:[Blazegraph](../../delta/api/supervision-api.md#blazegraph) -and @ref:[composite views](../../delta/api/supervision-api.md#composite-views) and +Using the supervision endpoints for @ref:[Blazegraph](../delta/api/supervision-api.md#blazegraph) +and @ref:[composite views](../delta/api/supervision-api.md#composite-views) and the @link:[Data on disk guidance from Blazegraph](https://github.com/blazegraph/database/wiki/Hardware_Configuration#data-on-disk-sizing-guidance) gives some insights on how much storage can be assigned. diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md b/docs/src/main/paradox/docs/running-nexus/configuration/index.md similarity index 92% rename from docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md rename to docs/src/main/paradox/docs/running-nexus/configuration/index.md index 0a0c7db5dc..1e9b049526 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md +++ b/docs/src/main/paradox/docs/running-nexus/configuration/index.md @@ -20,7 +20,7 @@ In order to successfully run Nexus Delta there is a minimum set of configuration The configuration flag `akka.http.server.parsing.max-content-length` can be used to control the maximum payload size allowed for Nexus Delta resources. This value applies to all posted resources except for files. -### Postgres configuration +## Postgres configuration @link:[The `database` section](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/app/src/main/resources/app.conf#L23){ open=new } of the configuration defines the postgres specific configuration. As Nexus Delta uses three separate pools ('read', 'write', 'streaming'), it is recommended to set the host, port, database name, username, and password via the `app.defaults.database` field, as it will apply to all pools. It is however possible to accommodate more advanced setups by configuring each pool separately by changing its respective `app.database.{read|write|streaming}` fields. @@ -55,6 +55,43 @@ Nexus Delta uses a service account to perform automatic tasks under the hood. Ex @link:[The `service-account` section](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/app/src/main/resources/app.conf#L427){ open=new } of the configuration defines the service account configuration. +## Realm provisioning + +Realm provisioning allows to create one or several realm at startup. + +It is useful to start a new deployment with having to call the @ref:[realm API](../../delta/api/realms-api.md) to create those. + +Exemple: +```hocon +realms { + #... + + # To provision realms at startup + # Only the name and the OpenId config url are mandatory + provisioning { + enabled = true + realms { + my-realm = { + name = "My realm name" + open-id-config = "https://.../path/.well-known/openid-configuration" + logo = "https://bbp.epfl.ch/path/favicon.png" + accepted-audiences = ["audience1", "audience2"] + } + } + } + } +``` + +@@@ note { .warning } + +Realm provisioning will only create realms. +If a realm with the same identifier exists it will not be updated. + +@@@ + +@link:[The `realms.provisioning` section](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/app/src/main/resources/app.conf){ open=new } of the configuration defines the service account configuration. + + ## Automatic project provisioning Automatic project provisioning allows to create a dedicated project for users the first time they connect to Delta that is to @@ -77,7 +114,7 @@ When fetching a resource, Nexus Delta allows to return a redirection to its repr ## Projections configuration -Projections in Nexus Delta are asynchronous processes that can replay the event log and process this information. For more information on projections, please refer to the @ref:[Architecture page](../../../delta/architecture.md). +Projections in Nexus Delta are asynchronous processes that can replay the event log and process this information. For more information on projections, please refer to the @ref:[Architecture page](../../delta/architecture.md). @link:[The `projections` section](https://github.com/BlueBrain/nexus/blob/$git.branch$/delta/app/src/main/resources/app.conf#L281) of the configuration allows to configure the projections. @@ -89,7 +126,7 @@ Since 1.5.0, Nexus Delta supports plugins. Jar files present inside the local di Each plugin configuration is rooted under `plugins.{plugin_name}`. All plugins have a `plugins.{plugin_name}.priority` configuration flag used to determine the order in which the routes are handled in case of collisions. -For more information about plugins, please refer to the @ref:[Plugins page](../../../delta/plugins/index.md). +For more information about plugins, please refer to the @ref:[Plugins page](../../delta/plugins/index.md). ### Elasticsearch views plugin configuration diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/logback.xml b/docs/src/main/paradox/docs/running-nexus/configuration/logback.xml similarity index 100% rename from docs/src/main/paradox/docs/getting-started/running-nexus/configuration/logback.xml rename to docs/src/main/paradox/docs/running-nexus/configuration/logback.xml diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/delta.md b/docs/src/main/paradox/docs/running-nexus/delta.md similarity index 100% rename from docs/src/main/paradox/docs/getting-started/running-nexus/delta.md rename to docs/src/main/paradox/docs/running-nexus/delta.md diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/docker/delta.conf b/docs/src/main/paradox/docs/running-nexus/docker/delta.conf similarity index 100% rename from docs/src/main/paradox/docs/getting-started/running-nexus/docker/delta.conf rename to docs/src/main/paradox/docs/running-nexus/docker/delta.conf diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/docker/docker-compose.yaml b/docs/src/main/paradox/docs/running-nexus/docker/docker-compose.yaml similarity index 69% rename from docs/src/main/paradox/docs/getting-started/running-nexus/docker/docker-compose.yaml rename to docs/src/main/paradox/docs/running-nexus/docker/docker-compose.yaml index aad6c59b65..ce189a1c8b 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/docker/docker-compose.yaml +++ b/docs/src/main/paradox/docs/running-nexus/docker/docker-compose.yaml @@ -1,19 +1,21 @@ services: delta: depends_on: - - blazegraph - - elasticsearch - - postgres - image: bluebrain/nexus-delta:1.10.0-M13 + elasticsearch: + condition: service_healthy + blazegraph: + condition: service_healthy + postgres: + condition: service_started + image: bluebrain/nexus-delta:1.10.0 environment: DELTA_PLUGINS: "/opt/docker/plugins/" DELTA_EXTERNAL_CONF: "/config/delta.conf" KAMON_ENABLED: "false" - entrypoint: [ "bin/wait-for-it.sh", "-s", "-t", "0", "postgres:5432", "--", - "bin/wait-for-it.sh", "-s", "-t", "0", "elasticsearch:9200", "--", - "./bin/delta-app", - "-Xmx512m" - ] + entrypoint: + - '/bin/bash' + - '-c' + - '/opt/docker/bin/delta-app -Xmx512m' volumes: - ${PWD}/delta.conf:/config/delta.conf deploy: @@ -29,6 +31,11 @@ services: xpack.security.enabled: "true" ingest.geoip.downloader.enabled: "false" ELASTIC_PASSWORD: "password" + healthcheck: + test: [ "CMD", "curl", "-f", "http://elastic:password@localhost:9200/" ] + interval: 1s + timeout: 2s + retries: 60 ports: - "9200" deploy: @@ -48,6 +55,11 @@ services: JAVA_OPTS: "-Djava.awt.headless=true -XX:MaxDirectMemorySize=64m -Xmx512m -XX:+UseG1GC" ports: - "9999" + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9999/blazegraph/" ] + interval: 1s + timeout: 2s + retries: 60 deploy: resources: limits: @@ -56,7 +68,7 @@ services: web: depends_on: - delta - image: bluebrain/nexus-web:1.9.9 + image: bluebrain/nexus-web:1.10.0-M2-studio-open-tag-in-drawer environment: BASE_PATH: "/" HOST_NAME: "http://localhost" diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/docker/nginx.conf b/docs/src/main/paradox/docs/running-nexus/docker/nginx.conf similarity index 100% rename from docs/src/main/paradox/docs/getting-started/running-nexus/docker/nginx.conf rename to docs/src/main/paradox/docs/running-nexus/docker/nginx.conf diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/elasticsearch.md b/docs/src/main/paradox/docs/running-nexus/elasticsearch.md similarity index 73% rename from docs/src/main/paradox/docs/getting-started/running-nexus/elasticsearch.md rename to docs/src/main/paradox/docs/running-nexus/elasticsearch.md index 73585f9b73..9493475d67 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/elasticsearch.md +++ b/docs/src/main/paradox/docs/running-nexus/elasticsearch.md @@ -10,6 +10,14 @@ The Elasticsearch @link:[setup documentation](https://www.elastic.co/guide/en/el contains the necessary information on how to install and configure it, but recommendations on sizing the nodes and cluster are scarce because it depends on usage. +@link:[This section of the documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/how-to.html) and @link:[the blog](https://www.elastic.co/blog) also gives useful recommendations on how to run Elasticsearch. + +@@@ note { .info } +Nexus is currently tested with Elasticsearch 8.x but should still work with the 7.x series. + +To benefit from the latest features and bug fixes from Elastic, we recommend you to stick as much as possible to the latest releases. +@@@ + ## Running and monitoring **CPU:** @@ -35,15 +43,37 @@ Elasticsearch provides a cluster health endpoint giving the status of the cluste This is important to monitor as it indicates in the yellow case that some data is likely to become unavailable if more shards disappear. -When the status turns to red, a primary shard is missing preventing indexing on that shard and causing search +When the status turns to red, a primary shard is missing. This prevents indexing on that shard and causes search to return partial results. -**Number of shards:** +**Number of shards per node:** Every view in Nexus uses at least 2 shards (one for a primary and one for a replica) and Elasticsearch allows a maximum number of shards per node. So when the number of the projects and views grow, Elasticsearch may run out of available shards and any new project or -view creation will result in an error +view creation will result in an error. + +@@@ note { .info } +To solve this, an option can be to add another node on the cluster. + +Another option if used with caution can be to raise the maximum number of shards per node. + +This can be done via the elasticsearch configuration file: +```hocon +cluster.max_shards_per_node: 2000 +``` + +Of via a call to the API: +``` +PUT _cluster/settings +{ + "persistent": { + "cluster.routing.allocation.total_shards_per_node": 2000 + } +} +``` + +@@@ **Search and indexing performance:** @@ -61,5 +91,3 @@ to monitoring and purchase a license to have access to some monitoring and alert Another option allowing to push data in Prometheus is @link:[elasticsearch_exporter](https://github.com/prometheus-community/elasticsearch_exporter). - -// TODO add some link to Elasticsearch recommendations \ No newline at end of file diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/index.md b/docs/src/main/paradox/docs/running-nexus/index.md similarity index 97% rename from docs/src/main/paradox/docs/getting-started/running-nexus/index.md rename to docs/src/main/paradox/docs/running-nexus/index.md index c1c9448f71..5c8941dab3 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/index.md +++ b/docs/src/main/paradox/docs/running-nexus/index.md @@ -121,7 +121,7 @@ $ curl http://localhost/v1/version | jq #### Using Fusion -Fusion can be accessed by opening `http://localhost` in your web browser. You can start by creating an organization from the `http://localhost/admin` page. +Fusion can be accessed by opening `http://localhost` in your web browser. You can start by creating an organization from the `http://localhost/orgs` page. @@@ note { .warning } @@ -214,7 +214,7 @@ Details on how to deploy, run and monitor the different components are described ### Choice of hardware Depending on the target throughput, usage profiles and data volume the hardware specification can vary greatly; please -take a look at the @ref:[benchmarks section](../../delta/benchmarks/v1.4.2.md) to get an idea of what you should expect in terms +take a look at the @ref:[benchmarks section](../delta/benchmarks/v1.4.2.md) to get an idea of what you should expect in terms of throughput with various hardware configurations. When the usage profiles are unknown a couple of rules of thumb should narrow the scope: @@ -224,7 +224,7 @@ should narrow the scope: access, so: * prefer local storage over network storage for lower latency when doing IO, * prefer SSD over HDDs because random access speed is more important than sequential access, - * one exception is the file storage (@ref:[file resources](../../delta/api/files-api.md) which are stored as + * one exception is the file storage (@ref:[file resources](../delta/api/files-api.md) which are stored as binary blobs on the filesystem) where the network disks should not be a cause for concern, nor random access speed; this assumes that accessing attachments is not the at the top in the usage profile 2. Nexus Delta and most of the "off the shelf" products are built to run on top of the JVM which usually @@ -258,4 +258,4 @@ While different approaches are possible depending on how and where Nexus is depl Prometheus has been the historical solution for monitoring Nexus at BBP for its popularity and its versatility as it allows to monitor all components. -![Monitoring with Prometheus](../assets/prometheus-monitoring.png) \ No newline at end of file +![Monitoring with Prometheus](../getting-started/assets/prometheus-monitoring.png) \ No newline at end of file diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/postgresql.md b/docs/src/main/paradox/docs/running-nexus/postgresql.md similarity index 97% rename from docs/src/main/paradox/docs/getting-started/running-nexus/postgresql.md rename to docs/src/main/paradox/docs/running-nexus/postgresql.md index 1db7b1e561..2c4d216de4 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/postgresql.md +++ b/docs/src/main/paradox/docs/running-nexus/postgresql.md @@ -83,7 +83,7 @@ Some of those actions may fail as it relies on external components (like a Blaze which may be down at this moment. This table allows to save the failures in order to be able to a posteriori call -@ref:[the supervision healing endpoint](../../delta/api/supervision-api.md#project-healing). +@ref:[the supervision healing endpoint](../delta/api/supervision-api.md#project-healing). **project_last_updates:** Stores the last_update and last_ordering properties for the different projects. @@ -142,7 +142,7 @@ Low memory indicates swapping and degraded performance. **Storage:** There should be enough available space for PostgreSQL to operate properly. -As described in the @ref:[architecture section](../../delta/architecture.md) the generally adopted +As described in the @ref:[architecture section](../delta/architecture.md) the generally adopted persistence model is an EventSourced model in which the data store is used as an _append only_ store. This has implications to the total amount of disk used by the primary store. diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/search-configuration.md b/docs/src/main/paradox/docs/running-nexus/search-configuration.md similarity index 95% rename from docs/src/main/paradox/docs/getting-started/running-nexus/search-configuration.md rename to docs/src/main/paradox/docs/running-nexus/search-configuration.md index 1162474ddc..70b9febf55 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/search-configuration.md +++ b/docs/src/main/paradox/docs/running-nexus/search-configuration.md @@ -20,10 +20,10 @@ There are several aspects that have been taken into consideration when adding gl Considering the requirements listed above, the implementation relies on existing Nexus features, namely: -- @ref:[Composite Views](../../delta/api/views/composite-view-api.md) as control resources for partitioning of indices, +- @ref:[Composite Views](../delta/api/views/composite-view-api.md) as control resources for partitioning of indices, how data is indexed, what information to collect, what permissions are required for querying -- @ref:[Automatic provisioning](../../delta/plugins/index.md#dependency-injection) of project resources after creation -- @ref:[Plugins](../../delta/plugins/index.md) for orchestrating the behaviour and exposing specific endpoints +- @ref:[Automatic provisioning](../delta/plugins/index.md#dependency-injection) of project resources after creation +- @ref:[Plugins](../delta/plugins/index.md) for orchestrating the behaviour and exposing specific endpoints When the search plugin is enabled and configured it will automatically create within each project a CompositeView that controls what resources are indexed and what information is collected for each resource. The reasons for using one @@ -43,18 +43,18 @@ CompositeViews have been chosen because they are quite versatile, support a wide - simple transformations More information about CompositeViews can be found in the -@ref:[API Reference](../../delta/api/views/composite-view-api.md). +@ref:[API Reference](../delta/api/views/composite-view-api.md). The search plugin introduces a new namespace (`/v1/search`) with two sub-resources (`query` and `config`). -The @ref:[query endpoint](../../delta/api/search-api.md#query) accepts submitting an +The @ref:[query endpoint](../delta/api/search-api.md#query) accepts submitting an @link:[Elasticsearch query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) via `POST`, similar to other views based on Elasticsearch, like `ElasticSearchView`, `AggregateElasticSearchView` or `CompositeView` with configured Elasticsearch projections. The query will be dispatched to all ElasticSearch indices managed by the CompositeViews created by the search plugin (the ones that share the id mentioned above) for which the client has access to. This ensures that access to information is restricted based on each project's access control. -The @ref:[config endpoint](../../delta/api/search-api.md#configuration) allows clients to discover the underlying index +The @ref:[config endpoint](../delta/api/search-api.md#configuration) allows clients to discover the underlying index data model such that it can present users (like in the case of Fusion) an appropriate interface for querying, filtering, sorting, aggregations etc. A minimal response for the config endpoint is like the following example: @@ -131,7 +131,7 @@ the rest of the search configuration: The search plugin must also be enabled using the `plugins.search.enabled=true` setting. These additional settings pertain to the configuration of the CompositeViews that are automatically provisioned by the -search plugin. The @ref:[CompositeView API Reference](../../delta/api/views/composite-view-api.md) provides a detailed +search plugin. The @ref:[CompositeView API Reference](../delta/api/views/composite-view-api.md) provides a detailed explanation on how CompositeViews work and how these options affect the generation of the indices. ## Example use case @@ -141,7 +141,7 @@ uses four related data types (`Dataset`, `DataDownload`, `Person` and `License`) ability to query resources of type `Dataset` along with information registered in related resources (of type `DataDownload`, `Person` or `License`). -[![Data Model](../assets/search-use-case.png "Data Model")](../assets/search-use-case.png) +[![Data Model](../getting-started/assets/search-use-case.png "Data Model")](../getting-started/assets/search-use-case.png) There are a couple of things to notice in the data model diagram: diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index b3f26954a1..0e028ae570 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -41,7 +41,8 @@ ship { s-3 { endpoint = "https://s3.us-east-1.amazonaws.com" # the bucket which contains the import files - import-bucket = "nexus-ship-production" + import-bucket = "default-import-bucket" + import-bucket = ${?IMPORT_BUCKET} } @@ -67,6 +68,16 @@ ship { name = "Default Sparql view" description = "A Sparql view of all resources in the project." } + + search { + name = "Default global search view" + description = "An Elasticsearch view of configured resources for the global search." + } + } + + search { + commit = "master" + rebuild-interval = 10 minutes } organizations { @@ -109,7 +120,8 @@ ship { show-location = true # the default maximum allowed file size (in bytes) for uploaded files. 10 GB default-max-file-size = 10737418240 - default-bucket = "nexus-delta-production" + default-bucket = "default-target-bucket" + default-bucket = ${?TARGET_BUCKET} } # Disk storage configuration @@ -173,7 +185,8 @@ ship { #prefix = "/some-prefix" import-bucket = ${ship.s-3.import-bucket} # The bucket to which the files will be copied by the Nexus Ship - target-bucket = "nexus-delta-production" + target-bucket = "default-target-bucket" + target-bucket = ${?TARGET_BUCKET} # To skip file events to make the batch run faster and focus on other events skip-file-events = false # Allows to define default media types for the given file extensions diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala index 36d9a5d4d3..d860086f31 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala @@ -19,6 +19,7 @@ final case class InputConfig( organizations: OrganizationCreationConfig, projectMapping: ProjectMapping = Map.empty, viewDefaults: ViewDefaults, + search: SearchConfig, serviceAccount: ServiceAccountConfig, storages: StoragesConfig, files: FileProcessingConfig, diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala new file mode 100644 index 0000000000..110ce5f3cc --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala @@ -0,0 +1,13 @@ +package ch.epfl.bluebrain.nexus.ship.config + +import pureconfig.ConfigReader +import pureconfig.generic.semiauto.deriveReader + +import scala.concurrent.duration.FiniteDuration + +final case class SearchConfig(commit: String, rebuildInterval: FiniteDuration) + +object SearchConfig { + implicit val searchConfigReader: ConfigReader[SearchConfig] = + deriveReader[SearchConfig] +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala index 6c638c57a3..c2f7b4ae45 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala @@ -6,7 +6,8 @@ import pureconfig.generic.semiauto.deriveReader case class ViewDefaults( elasticsearch: Defaults, - blazegraph: Defaults + blazegraph: Defaults, + search: Defaults ) object ViewDefaults { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala index 00fc327414..d2cb280ebb 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala @@ -3,25 +3,24 @@ package ch.epfl.bluebrain.nexus.ship.files import akka.http.scaladsl.model.Uri import akka.http.scaladsl.model.Uri.Path import cats.effect.IO -import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.RetryStrategy.logError -import ch.epfl.bluebrain.nexus.delta.kernel.{Logger, RetryStrategy} import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils +import ch.epfl.bluebrain.nexus.delta.kernel.{Logger, RetryStrategy} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{CopyOptions, S3LocationGenerator, S3OperationResult} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.client.S3StorageClient +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{CopyOptions, S3LocationGenerator} +import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.ship.config.FileProcessingConfig import ch.epfl.bluebrain.nexus.ship.files.FileCopier.FileCopyResult import ch.epfl.bluebrain.nexus.ship.files.FileCopier.FileCopyResult.{FileCopySkipped, FileCopySuccess} -import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ import software.amazon.awssdk.services.s3.model.S3Exception import scala.concurrent.duration.DurationInt trait FileCopier { - def copyFile(project: ProjectRef, attributes: FileAttributes, forceContentType: Boolean): IO[FileCopyResult] + def copyFile(project: ProjectRef, attributes: FileAttributes): IO[FileCopyResult] } @@ -34,7 +33,7 @@ object FileCopier { private val copyRetryStrategy: RetryStrategy[S3Exception] = RetryStrategy.constant( 30.seconds, 10, - e => e.statusCode() >= 500 && e.statusCode() < 600, + e => e.statusCode() == 403 || (e.statusCode() >= 500 && e.statusCode() < 600), logError(logger, "s3Copy") ) @@ -55,7 +54,7 @@ object FileCopier { val importBucket = config.importBucket val targetBucket = config.targetBucket val locationGenerator = new S3LocationGenerator(config.prefix.getOrElse(Path.Empty)) - (project: ProjectRef, attributes: FileAttributes, forceContentType: Boolean) => + (project: ProjectRef, attributes: FileAttributes) => { val origin = attributes.path val patchedFileName = if (attributes.filename.isEmpty) "file" else attributes.filename @@ -73,15 +72,6 @@ object FileCopier { s3StorageClient.copyObjectMultiPart(importBucket, originKey, targetBucket, targetKey, copyOptions) } else s3StorageClient.copyObject(importBucket, originKey, targetBucket, targetKey, copyOptions) - }.flatMap { - case S3OperationResult.Success => IO.unit - case S3OperationResult.AlreadyExists => - IO.whenA(forceContentType) { - attributes.mediaType.traverse { mediaType => - logger.info(s"Patching to content type $mediaType for file $patchedFileName") >> - s3StorageClient.updateContentType(targetBucket, targetKey, mediaType) - }.void - } }.timed .flatMap { case (duration, _) => IO.whenA(duration > longCopyThreshold)( @@ -104,7 +94,6 @@ object FileCopier { }.retry(copyRetryStrategy) } - def apply(): FileCopier = (_: ProjectRef, attributes: FileAttributes, _: Boolean) => - IO.pure(FileCopySuccess(attributes.path)) + def apply(): FileCopier = (_: ProjectRef, attributes: FileAttributes) => IO.pure(FileCopySuccess(attributes.path)) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala index 7d29aad890..abc1510edc 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessor.scala @@ -5,12 +5,12 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.kernel.utils.FileUtils -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{Files, MediaTypeDetector} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.definition import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CancelEvent import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{FileNotFound, IncorrectRev, ResourceAlreadyExists} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{Files, MediaTypeDetector} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.LinkFileAction @@ -27,7 +27,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} import ch.epfl.bluebrain.nexus.ship._ import ch.epfl.bluebrain.nexus.ship.config.InputConfig import ch.epfl.bluebrain.nexus.ship.files.FileCopier.FileCopyResult.{FileCopySkipped, FileCopySuccess} -import ch.epfl.bluebrain.nexus.ship.files.FileProcessor.{forceMediaType, logger, patchMediaType} +import ch.epfl.bluebrain.nexus.ship.files.FileProcessor.{logger, patchMediaType} import ch.epfl.bluebrain.nexus.ship.files.FileWiring._ import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring.linkS3FileOperationOnly @@ -74,8 +74,7 @@ class FileProcessor private ( val newMediaType = patchMediaType(attrs.filename, attrs.mediaType) val newAttrs = e.attributes.copy(mediaType = newMediaType) val customMetadata = Some(getCustomMetadata(newAttrs)) - val fct = forceMediaType(attrs.mediaType, newMediaType) - fileCopier.copyFile(e.project, newAttrs, fct).flatMap { + fileCopier.copyFile(e.project, newAttrs).flatMap { case FileCopySuccess(newPath) => val linkRequest = FileLinkRequest(newPath, newMediaType, customMetadata) files @@ -88,8 +87,7 @@ class FileProcessor private ( val newMediaType = patchMediaType(attrs.filename, attrs.mediaType) val newAttrs = e.attributes.copy(mediaType = newMediaType) val customMetadata = Some(getCustomMetadata(newAttrs)) - val fct = forceMediaType(attrs.mediaType, newMediaType) - fileCopier.copyFile(e.project, newAttrs, fct).flatMap { + fileCopier.copyFile(e.project, newAttrs).flatMap { case FileCopySuccess(newPath) => val linkRequest = FileLinkRequest(newPath, newMediaType, customMetadata) files @@ -140,11 +138,6 @@ object FileProcessor { .map(ContentType(_, () => HttpCharsets.`UTF-8`)) .orElse(original) - def forceMediaType( - originalMediaType: Option[ContentType], - newMediaType: Option[ContentType] - ): Boolean = originalMediaType != newMediaType - private val noop = new EventProcessor[FileEvent] { override def resourceType: EntityType = Files.entityType diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala index 1e1baefcd3..70871f0f33 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala @@ -4,14 +4,16 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.ScopeInitializer +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, ScopeInitializationErrorStore} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.ship.EventClock import ch.epfl.bluebrain.nexus.ship.config.InputConfig +import ch.epfl.bluebrain.nexus.ship.search.SearchWiring import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring.s3StorageInitializer -import ch.epfl.bluebrain.nexus.ship.views.ViewWiring.{blazegraphViews, elasticSearchViews, viewInitializers} +import ch.epfl.bluebrain.nexus.ship.views.ViewWiring.{blazegraphViews, compositeViews, elasticSearchViews, viewInitializers} object ScopeInitializerWiring { @@ -21,14 +23,21 @@ object ScopeInitializerWiring { config: InputConfig, clock: EventClock, xas: Transactors - )(implicit jsonLdApi: JsonLdApi): IO[ScopeInitializer] = + )(implicit jsonLdApi: JsonLdApi, baseUri: BaseUri): IO[ScopeInitializer] = for { - esViews <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) - bgViews <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) - storages <- StorageWiring.storages(fetchContext, rcr, config, clock, xas) - storageInit <- s3StorageInitializer(storages, config) - allInits = viewInitializers(esViews, bgViews, config) + storageInit - errorStore = ScopeInitializationErrorStore(xas, clock) + esViews <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + bgViews <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + compositeViews <- compositeViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + searchInit <- SearchWiring.searchInitializer( + compositeViews, + config.serviceAccount.value, + config.search, + config.viewDefaults.search + ) + storages <- StorageWiring.storages(fetchContext, rcr, config, clock, xas) + storageInit <- s3StorageInitializer(storages, config) + allInits = viewInitializers(esViews, bgViews, config) + searchInit + storageInit + errorStore = ScopeInitializationErrorStore(xas, clock) } yield ScopeInitializer(allInits, errorStore) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala new file mode 100644 index 0000000000..d09832f385 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala @@ -0,0 +1,82 @@ +package ch.epfl.bluebrain.nexus.ship.search + +import cats.effect.IO +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeView.Interval +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.TemplateSparqlConstructQuery +import ch.epfl.bluebrain.nexus.delta.plugins.search.SearchScopeInitialization +import ch.epfl.bluebrain.nexus.delta.plugins.search.model.SearchConfig.IndexingConfig +import ch.epfl.bluebrain.nexus.delta.plugins.search.model.SearchConfigError.{InvalidJsonError, InvalidSparqlConstructQuery, LoadingFileError} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue.ContextObject +import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery +import ch.epfl.bluebrain.nexus.delta.sdk.Defaults +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.ServiceAccount +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri +import ch.epfl.bluebrain.nexus.delta.sourcing.model.IriFilter +import ch.epfl.bluebrain.nexus.ship.config.SearchConfig +import io.circe.parser.decode +import io.circe.{Decoder, JsonObject} + +import java.net.URI +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import scala.concurrent.duration._ +import scala.util.Try + +object SearchWiring { + + private val client = HttpClient.newHttpClient() + + private def githubPrefix(commit: String) = + s"https://raw.githubusercontent.com/BlueBrain/nexus/$commit/tests/docker/config/search" + + private def getAsString(url: String) = { + val request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build() + IO.fromEither( + Try(client.send(request, HttpResponse.BodyHandlers.ofString())).toEither.leftMap(LoadingFileError(url, _)) + ) + } + + private def loadExternalConfig[A: Decoder](url: String): IO[A] = + for { + content <- getAsString(url) + value <- IO.fromEither(decode[A](content.body()).leftMap { e => InvalidJsonError(url, e.getMessage) }) + } yield value + + private def loadSparqlQuery(url: String): IO[SparqlConstructQuery] = + for { + content <- getAsString(url) + value <- IO.fromEither(TemplateSparqlConstructQuery(content.body()).leftMap { e => + InvalidSparqlConstructQuery(url, e) + }) + } yield value + + private def indexingConfig(commit: String, rebuildInterval: FiniteDuration) = { + val prefix = githubPrefix(commit) + for { + resourceTypes <- loadExternalConfig[IriFilter](s"$prefix/resource-types.json") + mapping <- loadExternalConfig[JsonObject](s"$prefix/mapping.json") + settings <- loadExternalConfig[JsonObject](s"$prefix/settings.json") + query <- loadSparqlQuery(s"$prefix/construct-query.sparql") + context <- loadExternalConfig[JsonObject](s"$prefix/search-context.json") + } yield IndexingConfig( + resourceTypes, + mapping, + settings = Some(settings), + query = query, + context = ContextObject(context), + rebuildStrategy = Some(Interval(rebuildInterval)) + ) + } + + def searchInitializer( + compositeViews: CompositeViews, + serviceAccount: ServiceAccount, + config: SearchConfig, + defaults: Defaults + )(implicit baseUri: BaseUri): IO[SearchScopeInitialization] = + indexingConfig(config.commit, config.rebuildInterval).map { config => + new SearchScopeInitialization(compositeViews, config, serviceAccount, defaults) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala index ee4e1172c5..1b2aca75ce 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent._ @@ -85,7 +86,7 @@ object CompositeViewProcessor { )(implicit jsonLdApi: JsonLdApi ): CompositeViewProcessor = { - val views = ViewWiring.compositeViews(fetchContext, rcr, config, clock, xas) + val views = (uuid: UUID) => ViewWiring.compositeViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas) new CompositeViewProcessor(views, projectMapper, clock) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala index 3be2bd1b4d..cf180e613c 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala @@ -82,21 +82,21 @@ object ViewWiring { rcr: ResolverContextResolution, config: EventLogConfig, clock: EventClock, + uuidF: UUIDF, xas: Transactors )(implicit jsonLdApi: JsonLdApi) = { val noValidation = new ValidateCompositeView { override def apply(uuid: UUID, value: CompositeViewValue): IO[Unit] = IO.unit } - (uuid: UUID) => - CompositeViews( - fetchContext, - rcr, - noValidation, - 3.seconds, // TODO: use the config? - config, - xas, - clock - )(jsonLdApi, UUIDF.fixed(uuid)) + CompositeViews( + fetchContext, + rcr, + noValidation, + 3.seconds, // TODO: use the config? + config, + xas, + clock + )(jsonLdApi, uuidF) } def viewInitializers( diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 44fcc436af..4f84257767 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -93,6 +93,7 @@ class RunShipSuite _ <- RunShip(events, s3Client, inputConfig, xas).assertEquals(expectedImportReport) _ <- checkFor("elasticsearch", nxv + "defaultElasticSearchIndex", xas).assertEquals(1) _ <- checkFor("blazegraph", nxv + "defaultSparqlIndex", xas).assertEquals(1) + _ <- checkFor("compositeviews", nxv + "searchView", xas).assertEquals(1) _ <- checkFor("storage", nxv + "defaultS3Storage", xas).assertEquals(1) } yield () } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala index 6d41615fde..dab74af7a4 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala @@ -8,6 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewType import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId => bgDefaultViewId} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewType.AggregateElasticSearch import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId => esDefaultViewId} +import ch.epfl.bluebrain.nexus.delta.plugins.search.model.{defaultViewId => searchViewId} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sourcing.exporter.ExportEventQuery @@ -53,6 +54,10 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { weFixThePermissions(project) thereShouldBeAProject(project, projectJson) + + thereShouldBeAViewWithId(project, bgDefaultViewId) + thereShouldBeAViewWithId(project, esDefaultViewId) + thereShouldBeAViewWithId(project, searchViewId) } "transfer multiple revisions of a project" in { @@ -185,6 +190,15 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { thereShouldBeAView(project, bgView, patchedSource) } + def thereShouldBeAViewWithId(project: ProjectRef, view: Iri): Assertion = { + val encodedIri = UrlUtils.encode(view.toString) + deltaClient + .get[Json](s"/views/${project.organization}/${project.project}/$encodedIri", writer) { (_, response) => + response.status shouldEqual StatusCodes.OK + } + .accepted + } + def thereShouldBeAView(project: ProjectRef, view: Iri, expectedJson: Json): Assertion = { val encodedIri = UrlUtils.encode(view.toString) deltaClient diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala index e07ec064fb..170038357f 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala @@ -14,6 +14,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, Defaults} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.testkit.scalatest.ClasspathResources +import concurrent.duration._ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with ClasspathResources { @@ -25,15 +26,16 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp private val viewDefaults = ViewDefaults( Defaults("Default ES View", "Description ES View"), - Defaults("Default EBG View", "Description BG View") + Defaults("Default EBG View", "Description BG View"), + Defaults("Default Search View", "Description Search View") ) private val serviceAccount: ServiceAccountConfig = ServiceAccountConfig( ServiceAccount(User("internal", Label.unsafe("sa"))) ) - private val importBucket = "nexus-ship-production" - private val targetBucket = "nexus-delta-production" + private val importBucket = "default-import-bucket" + private val targetBucket = "default-target-bucket" private val amazonConfig: S3StorageConfig = S3StorageConfig( @@ -57,6 +59,7 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp organizationsCreation, Map.empty, viewDefaults, + SearchConfig("master", 10.minutes), serviceAccount, StoragesConfig(eventLogConfig, pagination, config.copy(amazon = Some(amazonConfig))), FileProcessingConfig( diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessorSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessorSuite.scala index c4b1ebd0c4..769ab4ceec 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessorSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/files/FileProcessorSuite.scala @@ -25,24 +25,4 @@ class FileProcessorSuite extends NexusSuite { ) } - test("Patching media type for a media type that changes") { - assertEquals( - FileProcessor.forceMediaType( - Some(ContentTypes.`application/octet-stream`), - Some(ContentTypes.`application/json`) - ), - true - ) - } - - test("Patching media type for a media type that does not change") { - assertEquals( - FileProcessor.forceMediaType( - Some(ContentTypes.`application/json`), - Some(ContentTypes.`application/json`) - ), - false - ) - } - }