From 967cd61d73dbea733013f04001f37281d6053c04 Mon Sep 17 00:00:00 2001 From: bennettn4 Date: Thu, 14 Nov 2024 12:31:19 -0500 Subject: [PATCH 1/7] add azure gov azure environment --- .../azure/AzureConfiguration.scala | 23 +++++++++++++++++++ .../cloudsupport/azure/AzureCredentials.scala | 4 ++-- .../cloudsupport/azure/AzureUtils.scala | 2 +- core/src/main/resources/reference.conf | 5 ++++ .../filesystems/blob/BlobPathBuilder.scala | 8 ++++--- 5 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala new file mode 100644 index 00000000000..97181c52de2 --- /dev/null +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala @@ -0,0 +1,23 @@ +package cromwell.cloudsupport.azure + +import com.azure.core.management.AzureEnvironment +import com.typesafe.config.ConfigFactory +import net.ceedubs.ficus.Ficus._ + +object AzureConfiguration { + private val conf = ConfigFactory.load().getConfig("azure") + val azureEnvironment = AzureEnvironmentConverter.fromString(conf.as[Option[String]]("azure-environment").getOrElse("AZURE")) + val azureTokenScopeManagement = conf.as[String]("token-scope-management") +} + +object AzureEnvironmentConverter { + val Azure: String = "AZURE" + val AzureGov: String = "AZURE_GOV" + + def fromString(s: String): AzureEnvironment = s match { + case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT + // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default + case Azure => AzureEnvironment.AZURE + case _ => AzureEnvironment.AZURE + } +} diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala index d3d66e1bafc..67c429d06b8 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala @@ -20,8 +20,8 @@ case object AzureCredentials { final val tokenAcquisitionTimeout = 5.seconds - val azureProfile = new AzureProfile(AzureEnvironment.AZURE) - val tokenScope = "https://management.azure.com/.default" + val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment) + val tokenScope = AzureConfiguration.azureTokenScopeManagement private def tokenRequestContext: TokenRequestContext = { val trc = new TokenRequestContext() diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala index dd379ed3564..7fbfc303230 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala @@ -33,7 +33,7 @@ object AzureUtils { .map(Success(_)) .getOrElse(Failure(new Exception("Could not parse storage account"))) - val azureProfile = new AzureProfile(AzureEnvironment.AZURE) + val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment) def azureCredentialBuilder = new DefaultAzureCredentialBuilder() .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 9da59f16b7b..f7d5426ae3f 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -330,6 +330,11 @@ call-caching { max-failed-copy-attempts = 1000000 } +azure { + azure-environment = "AZURE" + token-scope-management = "https://management.azure.com/.default" +} + google { application-name = "cromwell" diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala index 3c984d1f2c8..c4cf9a1d53c 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala @@ -3,6 +3,7 @@ package cromwell.filesystems.blob import akka.http.scaladsl.model.Uri import com.azure.storage.blob.nio.AzureBlobFileAttributes import com.google.common.net.UrlEscapers +import cromwell.cloudsupport.azure.AzureConfiguration import cromwell.core.path.{NioPath, Path, PathBuilder} import cromwell.filesystems.blob.BlobPathBuilder._ @@ -13,13 +14,13 @@ import scala.language.postfixOps import scala.util.{Failure, Success, Try} object BlobPathBuilder { - private val blobHostnameSuffix = ".blob.core.windows.net" + private val blobHostnameSuffix = s".blob${AzureConfiguration.azureEnvironment.getStorageEndpointSuffix}" sealed trait BlobPathValidation case class ValidBlobPath(path: String, container: BlobContainerName, endpoint: EndpointURL) extends BlobPathValidation case class UnparsableBlobPath(errorMessage: Throwable) extends BlobPathValidation def invalidBlobHostMessage(endpoint: EndpointURL) = - s"Malformed Blob URL for this builder: The endpoint $endpoint doesn't contain the expected host string '{SA}.blob.core.windows.net/'" + s"Malformed Blob URL for this builder: The endpoint $endpoint doesn't contain the expected host string '{SA}.${blobHostnameSuffix}/'" def invalidBlobContainerMessage(endpoint: EndpointURL) = s"Malformed Blob URL for this builder: Could not parse container" val externalToken = @@ -103,7 +104,8 @@ object BlobPath { // 1) If the path starts with http:/ (single slash!) transform it to the containerName: // format the library expects // 2) If the path looks like :, strip off the : to leave the absolute path inside the container. - private val brokenPathRegex = "https:/([a-z0-9]+).blob.core.windows.net/([-a-zA-Z0-9]+)/(.*)".r + private val brokenPathRegex = + s"https:/([a-z0-9]+).blob${AzureConfiguration.azureEnvironment.getStorageEndpointSuffix}/([-a-zA-Z0-9]+)/(.*)".r // Blob files larger than 5 GB upload in parallel parts [0][1] and do not get a native `CONTENT-MD5` property. // Instead, some uploaders such as TES [2] calculate the md5 themselves and store it under this key in metadata. From 6d981cc5af12f5e590218f1b254a5cb2d071871f Mon Sep 17 00:00:00 2001 From: bennettn4 Date: Thu, 14 Nov 2024 12:42:56 -0500 Subject: [PATCH 2/7] Update AzureConfiguration.scala --- .../scala/cromwell/cloudsupport/azure/AzureConfiguration.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala index 97181c52de2..e74650adc4f 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala @@ -6,7 +6,8 @@ import net.ceedubs.ficus.Ficus._ object AzureConfiguration { private val conf = ConfigFactory.load().getConfig("azure") - val azureEnvironment = AzureEnvironmentConverter.fromString(conf.as[Option[String]]("azure-environment").getOrElse("AZURE")) + val azureEnvironment = + AzureEnvironmentConverter.fromString(conf.as[Option[String]]("azure-environment").getOrElse("AZURE")) val azureTokenScopeManagement = conf.as[String]("token-scope-management") } From 73cb567bf513c8c9a063538f8fe26d3fc8cf3988 Mon Sep 17 00:00:00 2001 From: bennettn4 Date: Thu, 14 Nov 2024 12:44:07 -0500 Subject: [PATCH 3/7] remove unused imports --- .../scala/cromwell/cloudsupport/azure/AzureCredentials.scala | 1 - .../src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala | 1 - 2 files changed, 2 deletions(-) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala index 67c429d06b8..034e464aa54 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala @@ -2,7 +2,6 @@ package cromwell.cloudsupport.azure import cats.implicits.catsSyntaxValidatedId import com.azure.core.credential.TokenRequestContext -import com.azure.core.management.AzureEnvironment import com.azure.core.management.profile.AzureProfile import com.azure.identity.DefaultAzureCredentialBuilder import common.validation.ErrorOr.ErrorOr diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala index 7fbfc303230..1febd5d77c0 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala @@ -1,6 +1,5 @@ package cromwell.cloudsupport.azure -import com.azure.core.management.AzureEnvironment import com.azure.core.management.profile.AzureProfile import com.azure.identity.DefaultAzureCredentialBuilder import com.azure.resourcemanager.AzureResourceManager From 84214c9d71721269f9945406d72e30a6f8aac514 Mon Sep 17 00:00:00 2001 From: bennettn4 Date: Mon, 18 Nov 2024 12:26:49 -0500 Subject: [PATCH 4/7] Update AzureConfiguration.scala --- .../cromwell/cloudsupport/azure/AzureConfiguration.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala index e74650adc4f..798443a2999 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala @@ -12,8 +12,8 @@ object AzureConfiguration { } object AzureEnvironmentConverter { - val Azure: String = "AZURE" - val AzureGov: String = "AZURE_GOV" + val Azure: String = "AzureCloud" + val AzureGov: String = "AzureUSGovernmentCloud" def fromString(s: String): AzureEnvironment = s match { case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT From ec6dfdf97d25cbdd03ee9b77accbb703a4d340e0 Mon Sep 17 00:00:00 2001 From: bennettn4 Date: Mon, 18 Nov 2024 13:54:20 -0500 Subject: [PATCH 5/7] Update AzureConfiguration.scala --- .../cromwell/cloudsupport/azure/AzureConfiguration.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala index 798443a2999..a82d35315c4 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala @@ -7,7 +7,9 @@ import net.ceedubs.ficus.Ficus._ object AzureConfiguration { private val conf = ConfigFactory.load().getConfig("azure") val azureEnvironment = - AzureEnvironmentConverter.fromString(conf.as[Option[String]]("azure-environment").getOrElse("AZURE")) + AzureEnvironmentConverter.fromString( + conf.as[Option[String]]("azure-environment").getOrElse(AzureEnvironmentConverter.Azure) + ) val azureTokenScopeManagement = conf.as[String]("token-scope-management") } From 03979c009569c352eb698effb9bfedc4067abd5f Mon Sep 17 00:00:00 2001 From: Blair L Murri Date: Thu, 5 Dec 2024 11:22:51 -0800 Subject: [PATCH 6/7] Add remaining active national clouds (#7666) --- .../scala/cromwell/cloudsupport/azure/AzureConfiguration.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala index a82d35315c4..6a3d4acaa62 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala @@ -16,9 +16,11 @@ object AzureConfiguration { object AzureEnvironmentConverter { val Azure: String = "AzureCloud" val AzureGov: String = "AzureUSGovernmentCloud" + val AzureChina: String = "AzureChinaCloud" def fromString(s: String): AzureEnvironment = s match { case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT + case AzureChina => AzureEnvironment.AZURE_CHINA // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default case Azure => AzureEnvironment.AZURE case _ => AzureEnvironment.AZURE From 69a00e6f0fb3764de2a1c1b2f67493a9ec932ba9 Mon Sep 17 00:00:00 2001 From: bennettn4 Date: Fri, 13 Dec 2024 11:27:38 -0500 Subject: [PATCH 7/7] Update reference.conf --- core/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index f7d5426ae3f..057b20a9642 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -331,7 +331,7 @@ call-caching { } azure { - azure-environment = "AZURE" + azure-environment = "AzureCloud" token-scope-management = "https://management.azure.com/.default" }