Update the authentication process for remote storages (#4241)

* authenticate using keycloak

* refactor AuthTokenProvider

* use specific token error

* scalafmt

* clarify test usage for AuthTokenProvider

* cache the access tokens

* change migration code which was broken when merging master branch

* use label for realm

* rename config class to 'Credentials'

* use the realm http client

* add anonymous as an authentication method

* add legacy auth token method to auth options

* add docs

* rename variable

* credentials are not optional

* rename AuthMethod to Credentials

* TokenError -> AuthTokenError

* KeycloakAuthService -> OpenIdAuthService

* add logging to caching open id auth token provider

* fix comment

* pr feedback

* add release notes

* fix storage config

* check realm is not deprecated

* use ParsedToken

* remove KeyValueStore.create

* change AuthTokenProvider interface

* move docs

* docs change

* add scaladoc

* scalafmt

delta/plugins/storage/src/main/resources/storage.conf

@@ -52,7 +52,9 @@ plugins.storage {
       # the default endpoint
       default-endpoint = "http://localhost:8084/v1"
       # the default credentials for the endpoint
-      default-credentials = null
+      credentials {
+        type: "anonymous"
+      }
       # the default digest algorithm
       digest-algorithm = "SHA-256"
       # the default permission required in order to download a file from a remote disk storage
@@ -61,7 +63,7 @@ plugins.storage {
       default-write-permission = "files/write"
       # flag to decide whether or not to show the absolute location of the files in the metadata response
       show-location = true
-      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB 
+      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB
       default-max-file-size = 10737418240
       # Retry strategy for digest computation
       digest-computation = ${app.defaults.retry-strategy}

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala

@@ -15,7 +15,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.Sto
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.contexts.{storages => storageCtxId, storagesMetadata => storageMetaCtxId}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageAccess
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.AuthTokenProvider
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.routes.StoragesRoutes
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.schemas.{storage => storagesSchemaId}
@@ -25,6 +24,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteCon
 import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
 import ch.epfl.bluebrain.nexus.delta.sdk._
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials, OpenIdAuthService}
 import ch.epfl.bluebrain.nexus.delta.sdk.deletion.ProjectDeletionTask
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives
 import ch.epfl.bluebrain.nexus.delta.sdk.fusion.FusionConfig
@@ -37,6 +37,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext.ContextRejection
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
+import ch.epfl.bluebrain.nexus.delta.sdk.realms.Realms
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder
 import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
@@ -146,8 +147,12 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
 
   many[ResourceShift[_, _, _]].ref[Storage.Shift]
 
-  make[AuthTokenProvider].from { (cfg: StorageTypeConfig) =>
-    AuthTokenProvider(cfg)
+  make[OpenIdAuthService].from { (httpClient: HttpClient @Id("realm"), realms: Realms) =>
+    new OpenIdAuthService(httpClient, realms)
+  }
+
+  make[AuthTokenProvider].fromEffect { (authService: OpenIdAuthService) =>
+    AuthTokenProvider(authService)
   }
 
   make[Files]
@@ -226,8 +231,14 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
     (
         client: HttpClient @Id("storage"),
         as: ActorSystem[Nothing],
-        authTokenProvider: AuthTokenProvider
-    ) => new RemoteDiskStorageClient(client, authTokenProvider)(as.classicSystem)
+        authTokenProvider: AuthTokenProvider,
+        cfg: StorageTypeConfig
+    ) =>
+      new RemoteDiskStorageClient(
+        client,
+        authTokenProvider,
+        cfg.remoteDisk.map(_.credentials).getOrElse(Credentials.Anonymous)
+      )(as.classicSystem)
   }
 
   many[ServiceDependency].addSet {

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala

@@ -5,6 +5,7 @@ import cats.implicits.toBifunctorOps
 import ch.epfl.bluebrain.nexus.delta.kernel.{RetryStrategyConfig, Secret}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{AbsolutePath, DigestAlgorithm, StorageType}
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.Credentials
 import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
 import ch.epfl.bluebrain.nexus.delta.sdk.model.search.PaginationConfig
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission
@@ -198,7 +199,7 @@ object StoragesConfig {
   final case class RemoteDiskStorageConfig(
       digestAlgorithm: DigestAlgorithm,
       defaultEndpoint: BaseUri,
-      defaultCredentials: Option[Secret[String]],
+      credentials: Credentials,
       defaultReadPermission: Permission,
       defaultWritePermission: Permission,
       showLocation: Boolean,

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala

@@ -10,10 +10,10 @@ import akka.http.scaladsl.model.Uri.Path
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedFetchError
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.MoveFileRejection.UnexpectedMoveError
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, MoveFileRejection, SaveFileRejection}
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.AuthTokenProvider
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskStorageFileAttributes
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
 import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials}
 import ch.epfl.bluebrain.nexus.delta.sdk.circe.CirceMarshalling._
 import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClientError._
 import ch.epfl.bluebrain.nexus.delta.sdk.http.{HttpClient, HttpClientError}
@@ -32,8 +32,8 @@ import scala.concurrent.duration._
 /**
   * The client to communicate with the remote storage service
   */
-final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenProvider)(implicit
-    as: ActorSystem
+final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenProvider, credentials: Credentials)(
+    implicit as: ActorSystem
 ) {
   import as.dispatcher
 
@@ -58,7 +58,7 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP
     *   the storage bucket name
     */
   def exists(bucket: Label)(implicit baseUri: BaseUri): IO[HttpClientError, Unit] = {
-    getAuthToken().flatMap { authToken =>
+    getAuthToken(credentials).flatMap { authToken =>
       val endpoint = baseUri.endpoint / "buckets" / bucket.value
       val req      = Head(endpoint).withCredentials(authToken)
       client(req) {
@@ -82,7 +82,7 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP
       relativePath: Path,
       entity: BodyPartEntity
   )(implicit baseUri: BaseUri): IO[SaveFileRejection, RemoteDiskStorageFileAttributes] = {
-    getAuthToken().flatMap { authToken =>
+    getAuthToken(credentials).flatMap { authToken =>
       val endpoint      = baseUri.endpoint / "buckets" / bucket.value / "files" / relativePath
       val filename      = relativePath.lastSegment.getOrElse("filename")
       val multipartForm = FormData(BodyPart("file", entity, Map("filename" -> filename))).toEntity()
@@ -106,7 +106,7 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP
     *   the relative path to the file location
     */
   def getFile(bucket: Label, relativePath: Path)(implicit baseUri: BaseUri): IO[FetchFileRejection, AkkaSource] = {
-    getAuthToken().flatMap { authToken =>
+    getAuthToken(credentials).flatMap { authToken =>
       val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / relativePath
       client.toDataBytes(Get(endpoint).withCredentials(authToken)).mapError {
         case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) =>
@@ -129,7 +129,7 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP
       bucket: Label,
       relativePath: Path
   )(implicit baseUri: BaseUri): IO[FetchFileRejection, RemoteDiskStorageFileAttributes] = {
-    getAuthToken().flatMap { authToken =>
+    getAuthToken(credentials).flatMap { authToken =>
       val endpoint = baseUri.endpoint / "buckets" / bucket.value / "attributes" / relativePath
       client.fromJsonTo[RemoteDiskStorageFileAttributes](Get(endpoint).withCredentials(authToken)).mapError {
         case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) =>
@@ -156,7 +156,7 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP
       sourceRelativePath: Path,
       destRelativePath: Path
   )(implicit baseUri: BaseUri): IO[MoveFileRejection, RemoteDiskStorageFileAttributes] = {
-    getAuthToken().flatMap { authToken =>
+    getAuthToken(credentials).flatMap { authToken =>
       val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / destRelativePath
       val payload  = Json.obj("source" -> sourceRelativePath.toString.asJson)
       client.fromJsonTo[RemoteDiskStorageFileAttributes](Put(endpoint, payload).withCredentials(authToken)).mapError {

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala

@@ -14,13 +14,13 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejec
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType.{RemoteDiskStorage => RemoteStorageType}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.AuthTokenProvider
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageFixtures, Storages, StoragesConfig, StoragesStatistics}
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv
 import ch.epfl.bluebrain.nexus.delta.sdk.ConfigFixtures
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials}
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.FileResponse
 import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.{Caller, ServiceAccount}
@@ -69,8 +69,8 @@ class FilesSpec(docker: RemoteStorageDocker)
     implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped
     implicit val httpClient: HttpClient                  = HttpClient()(httpClientConfig, system, sc)
     implicit val caller: Caller                          = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm)))
-    implicit val authTokenProvider: AuthTokenProvider    = AuthTokenProvider.test
-    val remoteDiskStorageClient                          = new RemoteDiskStorageClient(httpClient, authTokenProvider)
+    implicit val authTokenProvider: AuthTokenProvider    = AuthTokenProvider.anonymousForTest
+    val remoteDiskStorageClient                          = new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous)
 
     val tag        = UserTag.unsafe("tag")
     val otherRead  = Permission.unsafe("other/read")

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala

@@ -12,7 +12,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutesSpec.fileMetadata
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType}
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.AuthTokenProvider
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContexts, permissions => storagesPermissions, StorageFixtures, Storages, StoragesConfig, StoragesStatistics}
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
@@ -23,6 +22,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteCon
 import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials}
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives
 import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy
@@ -53,8 +53,8 @@ class FilesRoutesSpec
   import akka.actor.typed.scaladsl.adapter._
   implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped
   val httpClient: HttpClient                           = HttpClient()(httpClientConfig, system, s)
-  val authTokenProvider: AuthTokenProvider             = AuthTokenProvider.test
-  val remoteDiskStorageClient                          = new RemoteDiskStorageClient(httpClient, authTokenProvider)
+  val authTokenProvider: AuthTokenProvider             = AuthTokenProvider.anonymousForTest
+  val remoteDiskStorageClient                          = new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous)
 
   // TODO: sort out how we handle this in tests
   implicit override def rcr: RemoteContextResolution =

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala

@@ -5,6 +5,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{Di
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageFields.{DiskStorageFields, RemoteDiskStorageFields, S3StorageFields}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{AbsolutePath, DigestAlgorithm}
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.Credentials.Anonymous
 import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
@@ -29,7 +30,7 @@ trait StorageFixtures extends TestHelpers with CirceLiteral {
   implicit val config: StorageTypeConfig = StorageTypeConfig(
     disk = DiskStorageConfig(diskVolume, Set(diskVolume,tmpVolume), DigestAlgorithm.default, permissions.read, permissions.write, showLocation = false, Some(5000), 50),
     amazon = Some(S3StorageConfig(DigestAlgorithm.default, Some("localhost"), Some(Secret(MinioDocker.RootUser)), Some(Secret(MinioDocker.RootPassword)), permissions.read, permissions.write, showLocation = false, 60)),
-    remoteDisk = Some(RemoteDiskStorageConfig(DigestAlgorithm.default, BaseUri("http://localhost", Label.unsafe("v1")), None, permissions.read, permissions.write, showLocation = false, 70, RetryStrategyConfig.AlwaysGiveUp)),
+    remoteDisk = Some(RemoteDiskStorageConfig(DigestAlgorithm.default, BaseUri("http://localhost", Label.unsafe("v1")), Anonymous, permissions.read, permissions.write, showLocation = false, 70, RetryStrategyConfig.AlwaysGiveUp)),
   )
   val diskFields        = DiskStorageFields(Some("diskName"), Some("diskDescription"), default = true, Some(tmpVolume), Some(Permission.unsafe("disk/read")), Some(Permission.unsafe("disk/write")), Some(1000), Some(50))
   val diskVal           = diskFields.toValue(config).get

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageAccessSpec.scala

@@ -9,6 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.permissions._
 import ch.epfl.bluebrain.nexus.delta.sdk.ConfigFixtures
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials}
 import ch.epfl.bluebrain.nexus.delta.sdk.http.{HttpClient, HttpClientConfig}
 import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
@@ -37,8 +38,9 @@ class RemoteDiskStorageAccessSpec(docker: RemoteStorageDocker)
   implicit private val sc: Scheduler                = Scheduler.global
   implicit private val httpConfig: HttpClientConfig = httpClientConfig
   private val httpClient: HttpClient                = HttpClient()
-  private val authTokenProvider: AuthTokenProvider  = AuthTokenProvider.test
-  private val remoteDiskStorageClient               = new RemoteDiskStorageClient(httpClient, authTokenProvider)
+  private val authTokenProvider: AuthTokenProvider  = AuthTokenProvider.anonymousForTest
+  private val remoteDiskStorageClient               =
+    new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous)
 
   private val access = new RemoteDiskStorageAccess(remoteDiskStorageClient)
 

delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteStorageLinkFileSpec.scala

@@ -16,6 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.Storage
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.permissions.{read, write}
 import ch.epfl.bluebrain.nexus.delta.sdk.ConfigFixtures
+import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials}
 import ch.epfl.bluebrain.nexus.delta.sdk.http.{HttpClient, HttpClientConfig}
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, Tags}
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
@@ -47,8 +48,9 @@ class RemoteStorageLinkFileSpec(docker: RemoteStorageDocker)
   implicit val ec: ExecutionContext                 = system.dispatcher
   implicit private val httpConfig: HttpClientConfig = httpClientConfig
   private val httpClient: HttpClient                = HttpClient()
-  private val authTokenProvider: AuthTokenProvider  = AuthTokenProvider.test
-  private val remoteDiskStorageClient               = new RemoteDiskStorageClient(httpClient, authTokenProvider)
+  private val authTokenProvider: AuthTokenProvider  = AuthTokenProvider.anonymousForTest
+  private val remoteDiskStorageClient               =
+    new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous)
 
   private val iri      = iri"http://localhost/remote"
   private val uuid     = UUID.fromString("8049ba90-7cc6-4de5-93a1-802c04200dcc")