From d128eb279bd0a3735ba2556708ff15f6c9122a2b Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 7 Nov 2022 20:38:18 -0500 Subject: [PATCH 01/17] Dependencies --- project/Dependencies.scala | 8 +++++++- project/Publishing.scala | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8906e010531..7642018a34f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -197,6 +197,12 @@ object Dependencies { "com.azure.resourcemanager" % "azure-resourcemanager" % "2.18.0" ) + val wsmDependencies: List[ModuleID] = List( + "bio.terra" % "workspace-manager-client" % "0.254.441-SNAPSHOT" + exclude("jakarta.xml.bind", "jakarta.xml.bind-api") + exclude("jakarta.activation", "jakarta.activation-api") + ) + val implFtpDependencies = List( "commons-net" % "commons-net" % commonNetV, "io.github.andrebeat" %% "scala-pool" % scalaPoolV, @@ -407,7 +413,7 @@ object Dependencies { List("scalatest", "mysql", "mariadb", "postgresql") .map(name => "com.dimafeng" %% s"testcontainers-scala-$name" % testContainersScalaV % Test) - val blobFileSystemDependencies: List[ModuleID] = azureDependencies + val blobFileSystemDependencies: List[ModuleID] = azureDependencies ++ wsmDependencies val s3FileSystemDependencies: List[ModuleID] = junitDependencies diff --git a/project/Publishing.scala b/project/Publishing.scala index 1b798780086..1d4143fc13e 100644 --- a/project/Publishing.scala +++ b/project/Publishing.scala @@ -150,6 +150,10 @@ object Publishing { "Broad Artifactory" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/" + private val broadArtifactoryResolverSnap: Resolver = + "Broad Artifactory Snapshots" at + "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot-local/" + // https://stackoverflow.com/questions/9819965/artifactory-snapshot-filename-handling private val buildTimestamp = System.currentTimeMillis() / 1000 @@ -159,6 +163,7 @@ object Publishing { val additionalResolvers = List( broadArtifactoryResolver, + broadArtifactoryResolverSnap, Resolver.sonatypeRepo("releases") ) From b8549497de26899dc3bfd62118ecfca4356f897c Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 14 Nov 2022 12:48:45 -0500 Subject: [PATCH 02/17] Compiles but no tests --- .../blob/BlobFileSystemManager.scala | 37 +++++++++---------- .../blob/BlobPathBuilderFactory.scala | 12 +++++- .../WorkspaceManagerApiClientProvider.scala | 32 ++++++++++++++++ 3 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 3ebce4db878..3c12564029f 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -19,6 +19,8 @@ import scala.util.{Failure, Success, Try} import com.azure.resourcemanager.storage.models.StorageAccountKey import com.typesafe.scalalogging.LazyLogging +import java.util.UUID + case class FileSystemAPI() { def getFileSystem(uri: URI): Try[FileSystem] = Try(FileSystems.getFileSystem(uri)) def newFileSystem(uri: URI, config: Map[String, Object]): FileSystem = FileSystems.newFileSystem(uri, config.asJava) @@ -82,31 +84,26 @@ case class BlobFileSystemManager( sealed trait BlobTokenGenerator {def generateAccessToken: Try[AzureSasCredential]} object BlobTokenGenerator { def createBlobTokenGenerator(container: BlobContainerName, endpoint: EndpointURL, subscription: Option[SubscriptionId]): BlobTokenGenerator = { - createBlobTokenGenerator(container, endpoint, None, None, subscription) + NativeBlobTokenGenerator(container, endpoint, subscription) } - def createBlobTokenGenerator(container: BlobContainerName, - endpoint: EndpointURL, - workspaceId: Option[WorkspaceId], - workspaceManagerURL: Option[WorkspaceManagerURL], - subscription: Option[SubscriptionId] - ): BlobTokenGenerator = { - (container: BlobContainerName, endpoint: EndpointURL, workspaceId, workspaceManagerURL) match { - case (container, endpoint, None, None) => - NativeBlobTokenGenerator(container, endpoint, subscription) - case (container, endpoint, Some(workspaceId), Some(workspaceManagerURL)) => - WSMBlobTokenGenerator(container, endpoint, workspaceId, workspaceManagerURL) - case _ => - throw new Exception("Arguments provided do not match any available BlobTokenGenerator implementation.") - } + def createBlobTokenGenerator(container: BlobContainerName, endpoint: EndpointURL, workspaceId: WorkspaceId, workspaceManagerClient: WorkspaceManagerApiClientProvider): BlobTokenGenerator = { + WSMBlobTokenGenerator(container, endpoint, workspaceId, workspaceManagerClient) } - def createBlobTokenGenerator(container: BlobContainerName, endpoint: EndpointURL): BlobTokenGenerator = createBlobTokenGenerator(container, endpoint, None) - def createBlobTokenGenerator(container: BlobContainerName, endpoint: EndpointURL, workspaceId: Option[WorkspaceId], workspaceManagerURL: Option[WorkspaceManagerURL]): BlobTokenGenerator = - createBlobTokenGenerator(container, endpoint, workspaceId, workspaceManagerURL, None) } -case class WSMBlobTokenGenerator(container: BlobContainerName, endpoint: EndpointURL, workspaceId: WorkspaceId, workspaceManagerURL: WorkspaceManagerURL) extends BlobTokenGenerator { - def generateAccessToken: Try[AzureSasCredential] = Failure(new NotImplementedError) +case class WSMBlobTokenGenerator( + container: BlobContainerName, + endpoint: EndpointURL, + workspaceId: WorkspaceId, + wsmClient: WorkspaceManagerApiClientProvider) extends BlobTokenGenerator { + + def generateAccessToken: Try[AzureSasCredential] = Try { + val token = wsmClient.getControlledAzureResourceApi.createAzureStorageContainerSasToken( + UUID.fromString(workspaceId.value), + null, null, null, null, null).getToken // TODO `null` not OK, save for WX-696 + new AzureSasCredential(token) // TODO Does `signature` actually mean token? save for WX-696 + } } case class NativeBlobTokenGenerator(container: BlobContainerName, endpoint: EndpointURL, subscription: Option[SubscriptionId] = None) extends BlobTokenGenerator { diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala index 1a9df21b0f2..e0f7c992bb6 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala @@ -24,9 +24,17 @@ final case class BlobPathBuilderFactory(globalConfig: Config, instanceConfig: Co val workspaceId: Option[WorkspaceId] = instanceConfig.as[Option[String]]("workspace-id").map(WorkspaceId) val expiryBufferMinutes: Long = instanceConfig.as[Option[Long]]("expiry-buffer-minutes").getOrElse(10) val workspaceManagerURL: Option[WorkspaceManagerURL] = singletonConfig.config.as[Option[String]]("workspace-manager-url").map(WorkspaceManagerURL) + val b2cToken: Option[String] = instanceConfig.as[Option[String]]("b2cToken") + + val blobTokenGenerator: BlobTokenGenerator = (workspaceManagerURL, b2cToken, workspaceId) match { + case (Some(url), Some(token), Some(workspaceId)) => + val wsmClient: WorkspaceManagerApiClientProvider = new HttpWorkspaceManagerClientProvider(url, token) + // parameterizing client instead of URL to make injecting mock client possible + BlobTokenGenerator.createBlobTokenGenerator(container, endpoint, workspaceId, wsmClient) + case _ => + BlobTokenGenerator.createBlobTokenGenerator(container, endpoint, subscription) + } - val blobTokenGenerator: BlobTokenGenerator = BlobTokenGenerator.createBlobTokenGenerator( - container, endpoint, workspaceId, workspaceManagerURL, subscription) val fsm: BlobFileSystemManager = BlobFileSystemManager(container, endpoint, expiryBufferMinutes, blobTokenGenerator) override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[BlobPathBuilder] = { diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala new file mode 100644 index 00000000000..ea0b15d2f6a --- /dev/null +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala @@ -0,0 +1,32 @@ +package cromwell.filesystems.blob + +import bio.terra.workspace.api.ControlledAzureResourceApi +import bio.terra.workspace.client.ApiClient + +/** + * Represents a way to get various workspace manager clients + * + * Pared down from `org.broadinstitute.dsde.rawls.dataaccess.workspacemanager.WorkspaceManagerApiClientProvider` + * + * For testing, create an anonymous subclass as in `org.broadinstitute.dsde.rawls.dataaccess.workspacemanager.HttpWorkspaceManagerDAOSpec` + */ +trait WorkspaceManagerApiClientProvider { + def getApiClient: ApiClient + + def getControlledAzureResourceApi: ControlledAzureResourceApi + +} + +class HttpWorkspaceManagerClientProvider(baseWorkspaceManagerUrl: WorkspaceManagerURL, token: String) extends WorkspaceManagerApiClientProvider { + def getApiClient: ApiClient = { + val client: ApiClient = new ApiClient() + client.setBasePath(baseWorkspaceManagerUrl.value) + client.setAccessToken(token) + + client + } + + def getControlledAzureResourceApi: ControlledAzureResourceApi = + new ControlledAzureResourceApi(getApiClient) + +} \ No newline at end of file From 3f4b998bc51e27c62e3cf3d148cb25271a16ddaa Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 14 Nov 2022 12:55:49 -0500 Subject: [PATCH 03/17] Formatting --- .../cromwell/filesystems/blob/BlobFileSystemManager.scala | 8 +++++++- .../blob/WorkspaceManagerApiClientProvider.scala | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 3c12564029f..088e31c5b4b 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -101,7 +101,13 @@ case class WSMBlobTokenGenerator( def generateAccessToken: Try[AzureSasCredential] = Try { val token = wsmClient.getControlledAzureResourceApi.createAzureStorageContainerSasToken( UUID.fromString(workspaceId.value), - null, null, null, null, null).getToken // TODO `null` not OK, save for WX-696 + null, + null, + null, + null, + null + ).getToken // TODO `null` items may be required, investigate in WX-696 + new AzureSasCredential(token) // TODO Does `signature` actually mean token? save for WX-696 } } diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala index ea0b15d2f6a..c083d262151 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/WorkspaceManagerApiClientProvider.scala @@ -29,4 +29,4 @@ class HttpWorkspaceManagerClientProvider(baseWorkspaceManagerUrl: WorkspaceManag def getControlledAzureResourceApi: ControlledAzureResourceApi = new ControlledAzureResourceApi(getApiClient) -} \ No newline at end of file +} From 01a0dd1bfa4b43c0a8c4fe4c8c9dec122b7ef075 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 15 Nov 2022 11:50:37 -0500 Subject: [PATCH 04/17] Moar exclusions --- project/Dependencies.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7642018a34f..71627e9afc3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -201,6 +201,10 @@ object Dependencies { "bio.terra" % "workspace-manager-client" % "0.254.441-SNAPSHOT" exclude("jakarta.xml.bind", "jakarta.xml.bind-api") exclude("jakarta.activation", "jakarta.activation-api") + exclude("jakarta.activation", "jakarta.activation") + exclude("jakarta.annotation", "jakarta.annotation-api") + exclude("com.sun.activation", "jakarta.activation") + exclude("org.glassfish.hk2.external", "jakarta.inject") ) val implFtpDependencies = List( From 453aa64e3c51a651b9d47a27bedce0935949810e Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 15 Nov 2022 11:56:15 -0500 Subject: [PATCH 05/17] Update to latest WSM --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 71627e9afc3..6a5cf43d829 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -198,7 +198,7 @@ object Dependencies { ) val wsmDependencies: List[ModuleID] = List( - "bio.terra" % "workspace-manager-client" % "0.254.441-SNAPSHOT" + "bio.terra" % "workspace-manager-client" % "0.254.452-SNAPSHOT" exclude("jakarta.xml.bind", "jakarta.xml.bind-api") exclude("jakarta.activation", "jakarta.activation-api") exclude("jakarta.activation", "jakarta.activation") From b8a0f6429212112086d297e026b96cb185fbf47f Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Wed, 16 Nov 2022 15:23:07 -0500 Subject: [PATCH 06/17] Add additional dependency --- project/Dependencies.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6a5cf43d829..f1c56c21317 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -204,7 +204,8 @@ object Dependencies { exclude("jakarta.activation", "jakarta.activation") exclude("jakarta.annotation", "jakarta.annotation-api") exclude("com.sun.activation", "jakarta.activation") - exclude("org.glassfish.hk2.external", "jakarta.inject") + exclude("org.glassfish.hk2.external", "jakarta.inject"), + "org.glassfish.jersey.inject" % "jersey-hk2" % "2.32" ) val implFtpDependencies = List( From 10931f57cdcf763b2a77bf67db8bf3ba92f40f35 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Wed, 16 Nov 2022 15:23:21 -0500 Subject: [PATCH 07/17] We need some UUID here to make the request --- .../scala/cromwell/filesystems/blob/BlobFileSystemManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 088e31c5b4b..3c1cc2d2d55 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -101,7 +101,7 @@ case class WSMBlobTokenGenerator( def generateAccessToken: Try[AzureSasCredential] = Try { val token = wsmClient.getControlledAzureResourceApi.createAzureStorageContainerSasToken( UUID.fromString(workspaceId.value), - null, + UUID.fromString(workspaceId.value), // TODO replace with the real resource id null, null, null, From a135d48be09fa77ce465086fc8b6447251058786 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 16 Nov 2022 22:55:49 -0500 Subject: [PATCH 08/17] Formatting --- .../blob_md5/blob_md5.inputs.json | 3 ++ .../standardTestCases/blob_md5/blob_md5.wdl | 20 +++++++ .../blob/BlobFileSystemManager.scala | 2 +- .../blob/BlobPathBuilderFactory.scala | 4 +- project/Dependencies.scala | 3 +- ...Repo template_ Cromwell server TES.run.xml | 21 ++++++++ src/ci/resources/tes_application.conf | 53 ++++++++++++++++++- 7 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json create mode 100644 centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl create mode 100644 runConfigurations/Repo template_ Cromwell server TES.run.xml diff --git a/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json new file mode 100644 index 00000000000..c5a89ef0dbc --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json @@ -0,0 +1,3 @@ +{ + "fileChecksum.inputFile": "https://coaexternalstorage.blob.core.windows.net/cromwell/user-inputs/inputFile.txt" +} diff --git a/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl new file mode 100644 index 00000000000..b482f80d6e4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl @@ -0,0 +1,20 @@ +task md5 { + File inputFile + command { + echo "`date`: Running checksum on ${inputFile}..." + md5sum ${inputFile} > md5sum.txt + echo "`date`: Checksum is complete." + } + output { + File result = "md5sum.txt" + } + runtime { + docker: 'ubuntu:18.04' + preemptible: true + } +} + +workflow fileChecksum { + File inputFile + call md5 { input: inputFile=inputFile} +} diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala index 088e31c5b4b..877cefdfab5 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala @@ -101,7 +101,7 @@ case class WSMBlobTokenGenerator( def generateAccessToken: Try[AzureSasCredential] = Try { val token = wsmClient.getControlledAzureResourceApi.createAzureStorageContainerSasToken( UUID.fromString(workspaceId.value), - null, + UUID.fromString("00001111-2222-3333-aaaa-bbbbccccdddd"), null, null, null, diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala index e0f7c992bb6..70fc543ce9c 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala @@ -23,8 +23,8 @@ final case class BlobPathBuilderFactory(globalConfig: Config, instanceConfig: Co val endpoint: EndpointURL = EndpointURL(instanceConfig.as[String]("endpoint")) val workspaceId: Option[WorkspaceId] = instanceConfig.as[Option[String]]("workspace-id").map(WorkspaceId) val expiryBufferMinutes: Long = instanceConfig.as[Option[Long]]("expiry-buffer-minutes").getOrElse(10) - val workspaceManagerURL: Option[WorkspaceManagerURL] = singletonConfig.config.as[Option[String]]("workspace-manager-url").map(WorkspaceManagerURL) - val b2cToken: Option[String] = instanceConfig.as[Option[String]]("b2cToken") + val workspaceManagerURL: Option[WorkspaceManagerURL] = Option(WorkspaceManagerURL("example.com")) // singletonConfig.config.as[Option[String]]("workspace-manager-url").map(WorkspaceManagerURL) + val b2cToken: Option[String] = Option("asdf") // instanceConfig.as[Option[String]]("b2cToken") val blobTokenGenerator: BlobTokenGenerator = (workspaceManagerURL, b2cToken, workspaceId) match { case (Some(url), Some(token), Some(workspaceId)) => diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6a5cf43d829..b9d9509b7fc 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -204,7 +204,8 @@ object Dependencies { exclude("jakarta.activation", "jakarta.activation") exclude("jakarta.annotation", "jakarta.annotation-api") exclude("com.sun.activation", "jakarta.activation") - exclude("org.glassfish.hk2.external", "jakarta.inject") + exclude("org.glassfish.hk2.external", "jakarta.inject"), + "org.glassfish.jersey.inject" % "jersey-hk2" % "3.1.0", ) val implFtpDependencies = List( diff --git a/runConfigurations/Repo template_ Cromwell server TES.run.xml b/runConfigurations/Repo template_ Cromwell server TES.run.xml new file mode 100644 index 00000000000..9cb1818a004 --- /dev/null +++ b/runConfigurations/Repo template_ Cromwell server TES.run.xml @@ -0,0 +1,21 @@ + + + + \ No newline at end of file diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf index 018c13829d8..4e6f1d1e8b7 100644 --- a/src/ci/resources/tes_application.conf +++ b/src/ci/resources/tes_application.conf @@ -1,20 +1,69 @@ include required(classpath("application.conf")) include "build_application.inc.conf" +filesystems { + blob { + class = "cromwell.filesystems.blob.BlobPathBuilderFactory" + global { + class = "cromwell.filesystems.blob.BlobFileSystemConfig" + config { + workspace-manager-url: "https://workspace.dsde-dev.broadinstitute.org" + } + } + } +} + +engine { + filesystems { + local { + enabled: false + } + http { + enabled: false + } + blob { + enabled: true + container: "cromwell" + endpoint: "https://coaexternalstorage.blob.core.windows.net" + subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd" + b2cToken: "Zardoz" + } + } +} + + backend { default = "TES" providers { TES { actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" config { - root = "cromwell-executions" + root = "https://coaexternalstorage.blob.core.windows.net/cromwell/cromwell-executions" dockerRoot = "/cromwell-executions" endpoint = "http://127.0.0.1:9000/v1/tasks" concurrent-job-limit = 1000 + transform-blob-to-local-path = false + filesystems { + blob { + enabled: true + container: "cromwell" + endpoint: "https://coaexternalstorage.blob.core.windows.net" + subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + workspace-id: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + b2cToken: "Zardoz" + } + local { + enabled: false + } + http { + enabled: false + } + } } # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info. include "dockerhub_provider_config_v1.inc.conf" } } -} +} \ No newline at end of file From 94777edce78d472acc860b9ae3dc7d694b978534 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 16 Nov 2022 22:59:08 -0500 Subject: [PATCH 09/17] Clarify what is fake --- src/ci/resources/tes_application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf index 4e6f1d1e8b7..4580cdd0c4a 100644 --- a/src/ci/resources/tes_application.conf +++ b/src/ci/resources/tes_application.conf @@ -50,7 +50,7 @@ backend { container: "cromwell" endpoint: "https://coaexternalstorage.blob.core.windows.net" subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" - workspace-id: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd" b2cToken: "Zardoz" } local { From 6a6dbf9ba951650ff289e8730b84ec88d9c240d5 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 16 Nov 2022 23:00:14 -0500 Subject: [PATCH 10/17] Formatting --- src/ci/resources/tes_application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf index 4580cdd0c4a..139315a9c8b 100644 --- a/src/ci/resources/tes_application.conf +++ b/src/ci/resources/tes_application.conf @@ -66,4 +66,4 @@ backend { include "dockerhub_provider_config_v1.inc.conf" } } -} \ No newline at end of file +} From 3be7c3da1e83d290513cb8234955e7cb22e3f489 Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Thu, 17 Nov 2022 13:19:49 -0500 Subject: [PATCH 11/17] Use our own version of Jersey and Jackson stuff --- project/Dependencies.scala | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f1c56c21317..b90d0897019 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -197,15 +197,29 @@ object Dependencies { "com.azure.resourcemanager" % "azure-resourcemanager" % "2.18.0" ) + val jerseyVersion = "3.1.0" + val wsmDependencies: List[ModuleID] = List( - "bio.terra" % "workspace-manager-client" % "0.254.452-SNAPSHOT" + "bio.terra" % "workspace-manager-client" % "0.254.459-SNAPSHOT" exclude("jakarta.xml.bind", "jakarta.xml.bind-api") exclude("jakarta.activation", "jakarta.activation-api") exclude("jakarta.activation", "jakarta.activation") exclude("jakarta.annotation", "jakarta.annotation-api") exclude("com.sun.activation", "jakarta.activation") - exclude("org.glassfish.hk2.external", "jakarta.inject"), - "org.glassfish.jersey.inject" % "jersey-hk2" % "2.32" + exclude("org.glassfish.hk2.external", "jakarta.inject") + exclude("org.glassfish.jersey.core", "jersey-client") + exclude("org.glassfish.jersey.inject", "jersey-hk2") + exclude("org.glassfish.jersey.media", "jersey-media-json-jackson") + exclude("org.glassfish.jersey.media", "jersey-media-multipart") + exclude("javax.ws.rs", "javax.ws.rs-api") + exclude("com.fasterxml.jackson.datatype", "jackson-datatype-jsr310") + exclude("com.fasterxml.jackson.core", "jackson-databind"), + "org.glassfish.jersey.core" % "jersey-client" % jerseyVersion, + "org.glassfish.jersey.inject" % "jersey-hk2" % jerseyVersion, + "org.glassfish.jersey.media" % "jersey-media-json-jackson" % jerseyVersion, + "org.glassfish.jersey.media" % "jersey-media-multipart" % jerseyVersion, + "javax.ws.rs" % "javax.ws.rs-api" % "2.1.1", + "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonV ) val implFtpDependencies = List( From f38bb2c5eea7129d934b1784483fd471939a0610 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Fri, 18 Nov 2022 13:30:48 -0500 Subject: [PATCH 12/17] Port-in Khalid's changes (thank you!) Co-authored-by: Khalid Shakir --- .../blob/BlobPathBuilderFactory.scala | 2 +- .../blob/BlobPathBuilderFactorySpec.scala | 25 +++- project/Dependencies.scala | 132 +++++++++++++++--- 3 files changed, 134 insertions(+), 25 deletions(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala index 70fc543ce9c..d63f7bad2bc 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala @@ -23,7 +23,7 @@ final case class BlobPathBuilderFactory(globalConfig: Config, instanceConfig: Co val endpoint: EndpointURL = EndpointURL(instanceConfig.as[String]("endpoint")) val workspaceId: Option[WorkspaceId] = instanceConfig.as[Option[String]]("workspace-id").map(WorkspaceId) val expiryBufferMinutes: Long = instanceConfig.as[Option[Long]]("expiry-buffer-minutes").getOrElse(10) - val workspaceManagerURL: Option[WorkspaceManagerURL] = Option(WorkspaceManagerURL("example.com")) // singletonConfig.config.as[Option[String]]("workspace-manager-url").map(WorkspaceManagerURL) + val workspaceManagerURL: Option[WorkspaceManagerURL] = singletonConfig.config.as[Option[String]]("workspace-manager-url").map(WorkspaceManagerURL) val b2cToken: Option[String] = Option("asdf") // instanceConfig.as[Option[String]]("b2cToken") val blobTokenGenerator: BlobTokenGenerator = (workspaceManagerURL, b2cToken, workspaceId) match { diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala index f5b71a06ead..ed2090a73a3 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala @@ -27,7 +27,14 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga it should "parse configs for a functioning factory" in { val endpoint = BlobPathBuilderSpec.buildEndpoint("storageAccount") val container = BlobContainerName("storageContainer") - val workspaceId = WorkspaceId("mockWorkspaceId") + + + // Use a real UUID to help along the hacky "unit" test below. + //val workspaceId = WorkspaceId("mockWorkspaceId") + val workspaceId = WorkspaceId("B0BAFE77-0000-0000-0000-000000000000") + + + val workspaceManagerURL = WorkspaceManagerURL("https://test.ws.org") val instanceConfig = ConfigFactory.parseString( s""" @@ -45,6 +52,22 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga factory.expiryBufferMinutes should equal(10L) factory.workspaceId should contain(workspaceId) factory.workspaceManagerURL should contain(workspaceManagerURL) + + // Hacky "unit" test to try and exercise this branch's WSM code and dependencies. + // Should probably be in a Spec that extends TestKitSuite which provides and cleans up an ActorSystem. + import akka.actor.ActorSystem + import cromwell.core.WorkflowOptions + import scala.concurrent.{Await, ExecutionContext} + + implicit val system: ActorSystem = ActorSystem("BlobPathBuilderFactorySpec") + implicit val ec: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global + val pathBuilder = + Await.result(factory.withOptions(WorkflowOptions.empty), scala.concurrent.duration.Duration(10, "seconds")) + val sizeTry = pathBuilder.build(s"$endpoint/$container/inputs/test/testFile.wdl").map(_.size) + val sizeFailure = sizeTry.failed.get + sizeFailure shouldBe a[javax.ws.rs.ProcessingException] + sizeFailure.getMessage should include("unable to find valid certification path to requested target") + Await.result(system.terminate(), scala.concurrent.duration.Duration(10, "seconds")) } it should "build an example sas token of the correct format" in { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b90d0897019..c7bf0f759d2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -59,7 +59,12 @@ object Dependencies { private val hsqldbV = "2.6.1" private val http4sV = "0.21.31" // this release is EOL. We need to upgrade further for cats3. https://http4s.org/versions/ private val jacksonV = "2.13.3" + private val jakartaActivationV = "1.2.1" + private val jakartaAnnotationV = "1.3.5" + private val jakartaInjectV = "2.6.1" + private val jakartaXmlBindApiV = "2.3.2" private val janinoV = "3.1.7" + private val jerseyV = "2.32" // Use a jersey compatible with WSM. See notes in wsmDependencies below. private val jsr305V = "3.0.2" private val junitV = "4.13.2" private val kindProjectorV = "0.13.2" @@ -172,6 +177,16 @@ object Dependencies { "com.google.api-client" % "google-api-client-jackson2" % googleApiClientV exclude("com.google.guava", "guava-jdk5"), "com.google.cloud" % "google-cloud-resourcemanager" % googleCloudResourceManagerV, + /* + The google-cloud-java dependencies have similar issues with using an older javax.* vs. jakarta.* as guice. + google-cloud-java is still using javax.annotation and guice is sticking with javax.inject: + - https://github.com/google/guice/issues/1383 + - https://github.com/googleapis/google-cloud-java/blob/v0.201.0/google-cloud-jar-parent/pom.xml#L131-L136 + + Globally use of jakarta instead of javax until Google does themselves. + The javax.annotation exclusion is below in cromwellExcludeDependencies. + */ + "jakarta.annotation" % "jakarta.annotation-api" % jakartaAnnotationV, ) val spiDependencies: List[ModuleID] = List( @@ -200,26 +215,38 @@ object Dependencies { val jerseyVersion = "3.1.0" val wsmDependencies: List[ModuleID] = List( - "bio.terra" % "workspace-manager-client" % "0.254.459-SNAPSHOT" - exclude("jakarta.xml.bind", "jakarta.xml.bind-api") - exclude("jakarta.activation", "jakarta.activation-api") - exclude("jakarta.activation", "jakarta.activation") - exclude("jakarta.annotation", "jakarta.annotation-api") - exclude("com.sun.activation", "jakarta.activation") - exclude("org.glassfish.hk2.external", "jakarta.inject") - exclude("org.glassfish.jersey.core", "jersey-client") - exclude("org.glassfish.jersey.inject", "jersey-hk2") - exclude("org.glassfish.jersey.media", "jersey-media-json-jackson") - exclude("org.glassfish.jersey.media", "jersey-media-multipart") - exclude("javax.ws.rs", "javax.ws.rs-api") - exclude("com.fasterxml.jackson.datatype", "jackson-datatype-jsr310") - exclude("com.fasterxml.jackson.core", "jackson-databind"), - "org.glassfish.jersey.core" % "jersey-client" % jerseyVersion, - "org.glassfish.jersey.inject" % "jersey-hk2" % jerseyVersion, - "org.glassfish.jersey.media" % "jersey-media-json-jackson" % jerseyVersion, - "org.glassfish.jersey.media" % "jersey-media-multipart" % jerseyVersion, - "javax.ws.rs" % "javax.ws.rs-api" % "2.1.1", - "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonV + "bio.terra" % "workspace-manager-client" % "0.254.452-SNAPSHOT" +// exclude("jakarta.xml.bind", "jakarta.xml.bind-api") +// exclude("jakarta.activation", "jakarta.activation-api") +// exclude("jakarta.activation", "jakarta.activation") +// exclude("jakarta.annotation", "jakarta.annotation-api") +// exclude("org.glassfish.hk2.external", "jakarta.inject") + exclude("com.sun.activation", "jakarta.activation"), + /* + 1. WSM is looking for the rs-api under javax.*. + + Jersey 3.x switched to jakarta.ws.rs-api 3.x. If one uses jakarta's rs-api, 3.x will automatically evict 2.x. + + However, jakarta's rs-api 2.x provides packages javax.* while 3.x provides jakarta.* instead. + - https://javadoc.io/doc/jakarta.ws.rs/jakarta.ws.rs-api/2.1.6/javax/ws/rs/package-summary.html + - https://javadoc.io/doc/jakarta.ws.rs/jakarta.ws.rs-api/3.1.0/jakarta.ws.rs/module-summary.html + + TODO: Perhaps coordinate with the WSM team to use the jakarta 3.x rs-api and jakarta.* instead of javax.*. + + 2. Use the exact version of jersey that WSM is using. + + Jersey libraries cannot be mixed and matched as the various modules cannot be mixed and matched. + For example jersey-client 2.32 is not compatible with jersey-common 2.37. + + If needed one may also explicitly enumerate the list of jersey artifacts and explicitly set the versions similar to + catsDepeendencies, akkaHttpDependencies, etc. + - https://broadinstitute.jfrog.io/ui/repos/tree/PomView/libs-snapshot-local/bio/terra/workspace-manager-client/0.254.452-SNAPSHOT/workspace-manager-client-0.254.452-20221114.190249-1.pom + - https://github.com/eclipse-ee4j/jersey/blob/2.32/core-client/src/main/java/org/glassfish/jersey/client/ClientExecutorProvidersConfigurator.java#L139 + - https://github.com/eclipse-ee4j/jersey/blob/2.37/core-client/src/main/java/org/glassfish/jersey/client/ClientExecutorProvidersConfigurator.java#L136-L137 + */ + "org.glassfish.jersey.inject" % "jersey-hk2" % jerseyV + exclude("com.sun.activation", "jakarta.activation"), + "jakarta.activation" % "jakarta.activation-api" % jakartaActivationV, ) val implFtpDependencies = List( @@ -281,7 +308,11 @@ object Dependencies { ) private val liquibaseDependencies = List( + // The XML bind API replacement below may be removed when this ticket is addressed: + // https://github.com/liquibase/liquibase/issues/2991 "org.liquibase" % "liquibase-core" % liquibaseV + exclude("javax.xml.bind", "jaxb-api"), + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jakartaXmlBindApiV, ) private val akkaDependencies = List( @@ -348,15 +379,24 @@ object Dependencies { private val googleCloudDependencies = List( "io.grpc" % "grpc-core" % grpcV, "com.google.guava" % "guava" % guavaV, + /* + The google-cloud-nio has the same problems with an ancient inject as guice: + - https://github.com/google/guice/issues/1383 + - https://github.com/googleapis/java-storage-nio/blob/v0.124.20/google-cloud-nio/pom.xml#L49-L53 + + Force use of jakarta instead of javax until Google does themselves. + */ "com.google.cloud" % "google-cloud-nio" % googleCloudNioV exclude("com.google.api.grpc", "grpc-google-common-protos") exclude("com.google.cloud.datastore", "datastore-v1-protos") + exclude("javax.inject", "javax.inject") exclude("org.apache.httpcomponents", "httpclient"), "org.broadinstitute.dsde.workbench" %% "workbench-google" % workbenchGoogleV exclude("com.google.apis", "google-api-services-genomics"), "org.apache.httpcomponents" % "httpclient" % apacheHttpClientV, "com.google.apis" % "google-api-services-cloudkms" % googleCloudKmsV - exclude("com.google.guava", "guava-jdk5") + exclude("com.google.guava", "guava-jdk5"), + "org.glassfish.hk2.external" % "jakarta.inject" % jakartaInjectV, ) ++ googleGenomicsV2Alpha1Dependency ++ googleLifeSciencesV2BetaDependency private val dbmsDependencies = List( @@ -456,6 +496,8 @@ object Dependencies { val languageFactoryDependencies = List( "com.softwaremill.sttp" %% "core" % sttpV, "com.softwaremill.sttp" %% "async-http-client-backend-cats" % sttpV + exclude("com.sun.activation", "javax.activation"), + "jakarta.activation" % "jakarta.activation-api" % jakartaActivationV, ) val draft2LanguageFactoryDependencies = List( @@ -477,10 +519,16 @@ object Dependencies { - https://www.slf4j.org/legacy.html#jclOverSLF4J */ val owlApiDependencies = List( + // This whole section (incl. javax->jakarta issues) is probably going to be removed when CWL support is removed + // For now, replace javax usage with jakarta. "net.sourceforge.owlapi" % "owlapi-distribution" % owlApiV + exclude("javax.inject", "javax.inject") + exclude("javax.xml.bind", "jaxb-api") exclude("org.apache.httpcomponents", "httpclient-osgi") exclude("org.apache.httpcomponents", "httpcore-osgi") exclude("org.slf4j", "jcl-over-slf4j"), + "org.glassfish.hk2.external" % "jakarta.inject" % jakartaInjectV, + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jakartaXmlBindApiV, "org.apache.httpcomponents" % "httpclient-cache" % apacheHttpClientV, "org.apache.httpcomponents" % "httpclient" % apacheHttpClientV ) @@ -505,7 +553,9 @@ object Dependencies { val coreDependencies: List[ModuleID] = List( "com.google.auth" % "google-auth-library-oauth2-http" % googleOauth2V, "com.chuusai" %% "shapeless" % shapelessV, + // NOTE: See scalameter comment under engineDependencies "com.storm-enroute" %% "scalameter" % scalameterV % Test + exclude("com.fasterxml.jackson.module", "jackson-module-scala_2.13") exclude("org.scala-lang.modules", "scala-xml_2.13"), "com.github.scopt" %% "scopt" % scoptV, ) ++ akkaStreamDependencies ++ configDependencies ++ catsDependencies ++ circeDependencies ++ @@ -532,9 +582,22 @@ object Dependencies { val engineDependencies: List[ModuleID] = List( "commons-codec" % "commons-codec" % commonsCodecV, "commons-io" % "commons-io" % commonsIoV, + /* + Maybe ScalaMeter should be used, but is anyone? + + For now keep its dependencies from breaking jackson for other libraries. If someone wants to use it they can + re-fight with dependency-hell at that point. + + Avoid: + "com.fasterxml.jackson.databind.JsonMappingException: Scala module 2.11.3 requires Jackson Databind + version >= 2.11.0 and < 2.12.0": + - https://scalameter.github.io/home/gettingstarted/0.7/sbt/index.html + - https://github.com/FasterXML/jackson-module-scala/blob/jackson-module-scala-2.11.3/src/main/scala/com/fasterxml/jackson/module/scala/JacksonModule.scala#L53-L62 + */ "com.storm-enroute" %% "scalameter" % scalameterV exclude("com.fasterxml.jackson.core", "jackson-databind") exclude("com.fasterxml.jackson.module", "jackson-module-scala") + exclude("com.fasterxml.jackson.module", "jackson-module-scala_2.13") exclude("org.scala-tools.testing", "test-interface") exclude("org.scala-lang.modules", "scala-xml_2.13"), "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV, @@ -550,11 +613,24 @@ object Dependencies { val serverDependencies: List[ModuleID] = slf4jBindingDependencies val cromiamDependencies: List[ModuleID] = List( + /* + sttp 1.x was last released in 2019 + See above comment regarding "cats-effect, fs2, http4s, and sttp" all needing to update together. + For now, replace sttp 1.x's com.sun.activation usage with the jakarta version. + + NOTE when upgrading: sttp 3.x no longer requires an async-http-client-backend-future so jakarta.activation can + probably be removed from the dependencies: + - https://sttp.softwaremill.com/en/v3/backends/future.html#using-async-http-client + - https://sttp.softwaremill.com/en/v2/backends/future.html#using-async-http-client + - https://sttp.softwaremill.com/en/v1/backends/asynchttpclient.html + */ "com.softwaremill.sttp" %% "core" % sttpV, - "com.softwaremill.sttp" %% "async-http-client-backend-future" % sttpV, + "com.softwaremill.sttp" %% "async-http-client-backend-future" % sttpV + exclude("com.sun.activation", "javax.activation"), "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingV, "org.broadinstitute.dsde.workbench" %% "workbench-model" % workbenchModelV, - "org.broadinstitute.dsde.workbench" %% "workbench-util" % workbenchUtilV + "org.broadinstitute.dsde.workbench" %% "workbench-util" % workbenchUtilV, + "jakarta.activation" % "jakarta.activation-api" % jakartaActivationV, ) ++ akkaHttpDependencies ++ swaggerUiDependencies ++ slf4jBindingDependencies val wes2cromwellDependencies: List[ModuleID] = coreDependencies ++ akkaHttpDependencies @@ -761,5 +837,15 @@ object Dependencies { val cromwellExcludeDependencies: List[ExclusionRule] = List( // Replaced with jcl-over-slf4j ExclusionRule("commons-logging", "commons-logging"), + /* + The google-cloud-java dependencies have similar issues with using an older javax.* vs. jakarta.* as guice. + google-cloud-java is still using javax.annotation and guice is sticking with javax.inject: + - https://github.com/google/guice/issues/1383 + - https://github.com/googleapis/google-cloud-java/blob/v0.201.0/google-cloud-jar-parent/pom.xml#L131-L136 + + Globally use of jakarta instead of javax until Google does themselves. + The jakarta.annotation inclusion is above in googleApiClientDependencies. + */ + ExclusionRule("javax.annotation", "javax.annotation-api"), ) } From b3fad380ad9925f6338879c4f056eaf326716acc Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Fri, 18 Nov 2022 14:43:04 -0500 Subject: [PATCH 13/17] Test longevity Don't break the test if someone decides to add a cert to `ws.org` --- .../filesystems/blob/BlobPathBuilderFactorySpec.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala index ed2090a73a3..c11a6819847 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala @@ -7,6 +7,7 @@ import org.mockito.Mockito._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.channels.UnresolvedAddressException import java.nio.file.{FileSystem, FileSystemNotFoundException} import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit @@ -28,14 +29,11 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga val endpoint = BlobPathBuilderSpec.buildEndpoint("storageAccount") val container = BlobContainerName("storageContainer") - // Use a real UUID to help along the hacky "unit" test below. //val workspaceId = WorkspaceId("mockWorkspaceId") val workspaceId = WorkspaceId("B0BAFE77-0000-0000-0000-000000000000") - - - val workspaceManagerURL = WorkspaceManagerURL("https://test.ws.org") + val workspaceManagerURL = WorkspaceManagerURL("https://wsm.example.com") val instanceConfig = ConfigFactory.parseString( s""" |container = "$container" @@ -66,7 +64,7 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga val sizeTry = pathBuilder.build(s"$endpoint/$container/inputs/test/testFile.wdl").map(_.size) val sizeFailure = sizeTry.failed.get sizeFailure shouldBe a[javax.ws.rs.ProcessingException] - sizeFailure.getMessage should include("unable to find valid certification path to requested target") + sizeFailure.getCause.getClass shouldBe classOf[UnresolvedAddressException] Await.result(system.terminate(), scala.concurrent.duration.Duration(10, "seconds")) } From 24d9c10f99368e8ac4073668d8cbd34c37ab5dd6 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Fri, 18 Nov 2022 15:11:27 -0500 Subject: [PATCH 14/17] Cleanup --- project/Dependencies.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c7bf0f759d2..6e4376e0fce 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -216,11 +216,6 @@ object Dependencies { val wsmDependencies: List[ModuleID] = List( "bio.terra" % "workspace-manager-client" % "0.254.452-SNAPSHOT" -// exclude("jakarta.xml.bind", "jakarta.xml.bind-api") -// exclude("jakarta.activation", "jakarta.activation-api") -// exclude("jakarta.activation", "jakarta.activation") -// exclude("jakarta.annotation", "jakarta.annotation-api") -// exclude("org.glassfish.hk2.external", "jakarta.inject") exclude("com.sun.activation", "jakarta.activation"), /* 1. WSM is looking for the rs-api under javax.*. From f7dc8ef5d5f169661bf790f9c15bccbfe25e1ba8 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Fri, 18 Nov 2022 15:16:02 -0500 Subject: [PATCH 15/17] Cleanup --- .../cromwell/filesystems/blob/BlobPathBuilderFactory.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala index d63f7bad2bc..e0f7c992bb6 100644 --- a/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala +++ b/filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilderFactory.scala @@ -24,7 +24,7 @@ final case class BlobPathBuilderFactory(globalConfig: Config, instanceConfig: Co val workspaceId: Option[WorkspaceId] = instanceConfig.as[Option[String]]("workspace-id").map(WorkspaceId) val expiryBufferMinutes: Long = instanceConfig.as[Option[Long]]("expiry-buffer-minutes").getOrElse(10) val workspaceManagerURL: Option[WorkspaceManagerURL] = singletonConfig.config.as[Option[String]]("workspace-manager-url").map(WorkspaceManagerURL) - val b2cToken: Option[String] = Option("asdf") // instanceConfig.as[Option[String]]("b2cToken") + val b2cToken: Option[String] = instanceConfig.as[Option[String]]("b2cToken") val blobTokenGenerator: BlobTokenGenerator = (workspaceManagerURL, b2cToken, workspaceId) match { case (Some(url), Some(token), Some(workspaceId)) => From 6b2abb89ac8b5e98975a9d62c8b40076907e8f3c Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Fri, 18 Nov 2022 17:11:44 -0500 Subject: [PATCH 16/17] Cleanup --- .../blob_md5/blob_md5.inputs.json | 2 +- .../blob/BlobPathBuilderFactorySpec.scala | 21 ------------------- project/Dependencies.scala | 2 -- src/ci/resources/tes_application.conf | 10 ++++----- 4 files changed, 6 insertions(+), 29 deletions(-) diff --git a/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json index c5a89ef0dbc..f004c8be647 100644 --- a/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json +++ b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json @@ -1,3 +1,3 @@ { - "fileChecksum.inputFile": "https://coaexternalstorage.blob.core.windows.net/cromwell/user-inputs/inputFile.txt" + "fileChecksum.inputFile": "https://.blob.core.windows.net/cromwell/user-inputs/inputFile.txt" } diff --git a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala index c11a6819847..0cad162c860 100644 --- a/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala +++ b/filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderFactorySpec.scala @@ -7,7 +7,6 @@ import org.mockito.Mockito._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.nio.channels.UnresolvedAddressException import java.nio.file.{FileSystem, FileSystemNotFoundException} import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit @@ -28,11 +27,7 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga it should "parse configs for a functioning factory" in { val endpoint = BlobPathBuilderSpec.buildEndpoint("storageAccount") val container = BlobContainerName("storageContainer") - - // Use a real UUID to help along the hacky "unit" test below. - //val workspaceId = WorkspaceId("mockWorkspaceId") val workspaceId = WorkspaceId("B0BAFE77-0000-0000-0000-000000000000") - val workspaceManagerURL = WorkspaceManagerURL("https://wsm.example.com") val instanceConfig = ConfigFactory.parseString( s""" @@ -50,22 +45,6 @@ class BlobPathBuilderFactorySpec extends AnyFlatSpec with Matchers with MockSuga factory.expiryBufferMinutes should equal(10L) factory.workspaceId should contain(workspaceId) factory.workspaceManagerURL should contain(workspaceManagerURL) - - // Hacky "unit" test to try and exercise this branch's WSM code and dependencies. - // Should probably be in a Spec that extends TestKitSuite which provides and cleans up an ActorSystem. - import akka.actor.ActorSystem - import cromwell.core.WorkflowOptions - import scala.concurrent.{Await, ExecutionContext} - - implicit val system: ActorSystem = ActorSystem("BlobPathBuilderFactorySpec") - implicit val ec: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global - val pathBuilder = - Await.result(factory.withOptions(WorkflowOptions.empty), scala.concurrent.duration.Duration(10, "seconds")) - val sizeTry = pathBuilder.build(s"$endpoint/$container/inputs/test/testFile.wdl").map(_.size) - val sizeFailure = sizeTry.failed.get - sizeFailure shouldBe a[javax.ws.rs.ProcessingException] - sizeFailure.getCause.getClass shouldBe classOf[UnresolvedAddressException] - Await.result(system.terminate(), scala.concurrent.duration.Duration(10, "seconds")) } it should "build an example sas token of the correct format" in { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6e4376e0fce..d3ba06edd55 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -212,8 +212,6 @@ object Dependencies { "com.azure.resourcemanager" % "azure-resourcemanager" % "2.18.0" ) - val jerseyVersion = "3.1.0" - val wsmDependencies: List[ModuleID] = List( "bio.terra" % "workspace-manager-client" % "0.254.452-SNAPSHOT" exclude("com.sun.activation", "jakarta.activation"), diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf index 139315a9c8b..037babdcf1b 100644 --- a/src/ci/resources/tes_application.conf +++ b/src/ci/resources/tes_application.conf @@ -24,8 +24,8 @@ engine { blob { enabled: true container: "cromwell" - endpoint: "https://coaexternalstorage.blob.core.windows.net" - subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + endpoint: "https://.blob.core.windows.net" + subscription: "00001111-2222-3333-aaaa-bbbbccccdddd" workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd" b2cToken: "Zardoz" } @@ -39,7 +39,7 @@ backend { TES { actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" config { - root = "https://coaexternalstorage.blob.core.windows.net/cromwell/cromwell-executions" + root = "https://.blob.core.windows.net/cromwell/cromwell-executions" dockerRoot = "/cromwell-executions" endpoint = "http://127.0.0.1:9000/v1/tasks" concurrent-job-limit = 1000 @@ -48,8 +48,8 @@ backend { blob { enabled: true container: "cromwell" - endpoint: "https://coaexternalstorage.blob.core.windows.net" - subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + endpoint: "https://.blob.core.windows.net" + subscription: "00001111-2222-3333-aaaa-bbbbccccdddd" workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd" b2cToken: "Zardoz" } From 4c0373f24178b5f18894807e58673e9a4aea97eb Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Mon, 21 Nov 2022 10:00:16 -0500 Subject: [PATCH 17/17] Adjust TES config file for CI --- src/ci/resources/tes_application.conf | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf index 037babdcf1b..e6595c8e941 100644 --- a/src/ci/resources/tes_application.conf +++ b/src/ci/resources/tes_application.conf @@ -16,13 +16,13 @@ filesystems { engine { filesystems { local { - enabled: false + enabled: true } http { - enabled: false + enabled: true } blob { - enabled: true + enabled: false container: "cromwell" endpoint: "https://.blob.core.windows.net" subscription: "00001111-2222-3333-aaaa-bbbbccccdddd" @@ -39,14 +39,16 @@ backend { TES { actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory" config { - root = "https://.blob.core.windows.net/cromwell/cromwell-executions" + # Use for running on blob storage + #root = "https://.blob.core.windows.net/cromwell/cromwell-executions" + root = "cromwell-executions" dockerRoot = "/cromwell-executions" endpoint = "http://127.0.0.1:9000/v1/tasks" concurrent-job-limit = 1000 transform-blob-to-local-path = false filesystems { blob { - enabled: true + enabled: false container: "cromwell" endpoint: "https://.blob.core.windows.net" subscription: "00001111-2222-3333-aaaa-bbbbccccdddd" @@ -54,10 +56,10 @@ backend { b2cToken: "Zardoz" } local { - enabled: false + enabled: true } http { - enabled: false + enabled: true } } }