From cc0e240f742400ded89d9d9599ea335e30772c27 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 9 Jun 2022 22:38:48 +0000 Subject: [PATCH 01/17] Update cromwell version from 80 to 81 --- project/Version.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index b894fde796c..557c47dd458 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -5,7 +5,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on a master / hotfix branch - val cromwellVersion = "80" + val cromwellVersion = "81" /** * Returns true if this project should be considered a snapshot. From ded7e718e662d7dfb2f7ca6b0ae6d8c273f5c2b5 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:10:44 -0400 Subject: [PATCH 02/17] BW-1256 /runs/{run id} endpoint_KP (#6777) * Removed Wes2Cromwell project, added 'runs' endpoint a necessary functions. * saving... * saving... * saving... * saving... * Fix missing JSON implicits * Added endpoint to cromwell.yaml and added api prefix * Getting rid of unnecessary implicits * Fixing and changing function name * Updated RESTAPI.md, cromwell.yaml, and RunListResponse.scala * Getting rid of comments * Omitting unexpected arguments * Removing headers, adding further details in cromwell.yaml * Updating description and RESTAPI.md * saving... * saving... * saving... * saving... * saving... * Moving functions to consolidate, updating CHANGELOG.md, deleting Wes2CromwellInterface.scala file * Updated CHANGELOG.md, passed error message to API * saving... * Update CHANGELOG.md Co-authored-by: Adam Nichols * Update engine/src/main/scala/cromwell/webservice/routes/wes/WesRunRoutes.scala Co-authored-by: Adam Nichols * Update engine/src/main/scala/cromwell/webservice/routes/wes/WesRunRoutes.scala Co-authored-by: Adam Nichols * Changes to CHANGELOG.md, updating routes * Unused imports * Cleanup * Cleanup * codecov * saving... * saving... * saving... * saving... * runLog method * saving... * RESTAPI.md * Code cleanup * Code cleanup + comment * Unused import correction Co-authored-by: Adam Nichols Co-authored-by: Adam Nichols Co-authored-by: Saloni Shah --- docs/api/RESTAPI.md | 36 ++++++- .../src/main/resources/swagger/cromwell.yaml | 37 +++++++ .../routes/wes/CromwellMetadata.scala | 101 ++++++++++++++++++ .../routes/wes/RunListResponse.scala | 3 +- .../webservice/routes/wes/WesResponse.scala | 9 +- .../webservice/routes/wes/WesRunLog.scala | 34 ++++++ .../webservice/routes/wes/WesRunRoutes.scala | 65 ++++++----- 7 files changed, 255 insertions(+), 30 deletions(-) create mode 100644 engine/src/main/scala/cromwell/webservice/routes/wes/CromwellMetadata.scala create mode 100644 engine/src/main/scala/cromwell/webservice/routes/wes/WesRunLog.scala diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md index bdc5b6a15ff..b15072f6407 100644 --- a/docs/api/RESTAPI.md +++ b/docs/api/RESTAPI.md @@ -1,5 +1,5 @@ diff --git a/docs/developers/bitesize/ci/travis_centaur.md b/docs/developers/bitesize/ci/travis_centaur.md index 5d1bb836109..b56ec16c8af 100644 --- a/docs/developers/bitesize/ci/travis_centaur.md +++ b/docs/developers/bitesize/ci/travis_centaur.md @@ -7,7 +7,6 @@ Other backends run tests for any user. | Backend | Read-only users | Write/Admin users | |---------------|:---------------:|:-----------------:| | AWS | | ✅ | -| BCS | | ✅ | | Local | ✅ | ✅ | | PAPI V2alpha1 | | ✅ | | PAPI V2beta | | ✅ | @@ -64,7 +63,6 @@ or `papi_v2beta_centaur_application.conf` | Backend | MySQL | PostgreSQL | MariaDB | |---------|:------:|:-----------:|:--------:| | AWS | ✅ | | | -| BCS | ✅ | | | | Local | ✅ | ✅ | | | PAPI V2 | ✅ | | ⭕ | | SLURM | ✅ | | | diff --git a/docs/filesystems/Filesystems.md b/docs/filesystems/Filesystems.md index 641761fcfe1..d5a66de7eba 100644 --- a/docs/filesystems/Filesystems.md +++ b/docs/filesystems/Filesystems.md @@ -40,9 +40,6 @@ filesystems { gcs { class = "cromwell.filesystems.gcs.GcsPathBuilderFactory" } - oss { - class = "cromwell.filesystems.oss.OssPathBuilderFactory" - } s3 { class = "cromwell.filesystems.s3.S3PathBuilderFactory" } @@ -53,10 +50,10 @@ filesystems { ``` It defines the filesystems that can be accessed by Cromwell. -Those filesystems can be referenced by their name (`drs`, `gcs`, `oss`, `s3`, `http` and `local`) in other parts of the configuration. +Those filesystems can be referenced by their name (`drs`, `gcs`, `s3`, `http` and `local`) in other parts of the configuration. **Note:** -- **OSS and S3 filesystems are experimental.** +- **S3 filesystem is experimental.** - **DRS filesystem has initial support only. Also, currently it works only with [GCS filesystem](../GoogleCloudStorage) in [PapiV2 backend](http://cromwell.readthedocs.io/en/develop/backends/Google).** @@ -197,8 +194,6 @@ The filesystem configuration used will be the one in the `config` section of the - Simple Storage Service (S3) - [Amazon Doc](https://aws.amazon.com/documentation/s3/) -- Object Storage Service (OSS) - [Alibaba Cloud Doc](https://www.alibabacloud.com/product/oss) - - HTTP - support for `http` or `https` URLs for [workflow inputs only](http://cromwell.readthedocs.io/en/develop/filesystems/HTTP) - File Transfer Protocol (FTP) - [Cromwell Doc](FileTransferProtocol.md) diff --git a/docs/tutorials/BCSIntro.md b/docs/tutorials/BCSIntro.md deleted file mode 100644 index 11b55075a6d..00000000000 --- a/docs/tutorials/BCSIntro.md +++ /dev/null @@ -1,122 +0,0 @@ -## Getting started on Alibaba Cloud with the Batch Compute Service - -### Prerequisites - -This tutorial page relies on completing the previous tutorials: - -- [Configuration Files](ConfigurationFiles.md) - -### Goals - -In this tutorial you'll learn to run the first workflow against the Batch Compute service on Alibaba Cloud. - -### Let's get started! - -#### - -#### Configuring Alibaba Cloud - -- Go to Alibaba Cloud and activate Alibaba Cloud OSS and Alibaba Cloud BatchCompute services. -- Follow AccessKey Guide to retrieve an access-id and access-key pair. We will refer to this pair as `` and ``, respectively. -- Log on to the OSS console and choose a region to create a new bucket. We will use `` and `` to refer the chosen region and bucket. -- Find the corresponding OSS endpoint in OSS region and endpoint. We will refer to it as ``. - -#### Preparing workflow source files - -Copy over the sample `echo.wdl` and `echo.inputs` files to the same directory as the Cromwell jar. -This workflow takes a string value as an output file name and writes "Hello World!" to the file. - -***echo.wdl*** - -``` -task echo { - String out - - command { - echo Hello World! > ${out} - } - - output { - File outFile = "${out}" - Array[String] content = read_lines(outFile) - } -} - -workflow wf_echo { - call echo - output { - echo.outFile - echo.content - } -} -``` - -***echo.inputs*** - -``` -{ - "wf_echo.echo.out": "output" -} -``` - -#### Configuration file for Alibaba Cloud - -Copy over the sample `bcs.conf` file to the same directory that contains your sample WDL, inputs and the Cromwell jar. Replace ``, ``, ``, ``, `` in the configuration file with actual values. - -***bcs.conf*** - -``` -include required(classpath("application")) - -backend { - default = "BCS" - - providers { - BCS { - actor-factory = "cromwell.backend.impl.bcs.BcsBackendLifecycleActorFactory" - config { - root = "oss:///cromwell-dir" - region = "" - access-id = "" - access-key = "" - - filesystems { - oss { - auth { - endpoint = "" - access-id = "" - access-key = "" - } - } - } - - default-runtime-attributes { - failOnStderr: false - continueOnReturnCode: 0 - cluster: "OnDemand ecs.sn1ne.large img-ubuntu" - vpc: "192.168.0.0/16" - } - } - } - } -} -``` - -#### Run workflow - -`java -Dconfig.file=bcs.conf -jar cromwell.jar run echo.wdl --inputs echo.inputs` - -#### Outputs - -The end of your workflow logs should report the workflow outputs. - -``` -[info] SingleWorkflowRunnerActor workflow finished with status 'Succeeded'. -{ - "outputs": { - "wf_echo.echo.outFile": "oss:///cromwell-dir/wf_echo/38b088b2-5131-4ea0-a161-4cf2ca8d15ac/call-echo/output", - "wf_echo.echo.content": ["Hello World!"] - }, - "id": "38b088b2-5131-4ea0-a161-4cf2ca8d15ac" -} -``` diff --git a/docs/tutorials/Containers.md b/docs/tutorials/Containers.md index f565ced2332..3934f7a4579 100644 --- a/docs/tutorials/Containers.md +++ b/docs/tutorials/Containers.md @@ -509,8 +509,6 @@ Congratulations for improving the reproducibility of your workflows! You might f - [Getting started with AWS Batch](AwsBatch101.md) - [Getting started on Google Pipelines API](PipelinesApi101.md) -- [Getting started on Alibaba Cloud](BCSIntro/) - [cromwell-examples-conf]: https://www.github.com/broadinstitute/cromwell/tree/develop/cromwell.example.backends/cromwell.examples.conf [cromwell-examples-folder]: https://www.github.com/broadinstitute/cromwell/tree/develop/cromwell.example.backends diff --git a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala index a67df146abe..0598e154f2d 100644 --- a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala +++ b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala @@ -14,7 +14,6 @@ import cromwell.engine.io.RetryableRequestSupport.{isInfinitelyRetryable, isRetr import cromwell.engine.io.{IoAttempts, IoCommandContext, IoCommandStalenessBackpressuring} import cromwell.filesystems.drs.DrsPath import cromwell.filesystems.gcs.GcsPath -import cromwell.filesystems.oss.OssPath import cromwell.filesystems.s3.S3Path import cromwell.util.TryWithResource._ import net.ceedubs.ficus.Ficus._ @@ -166,9 +165,6 @@ class NioFlow(parallelism: Int, case s3Path: S3Path => IO { FileHash(HashType.S3Etag, s3Path.eTag) } - case ossPath: OssPath => IO { - FileHash(HashType.OssEtag, ossPath.eTag) - } case path => getMd5FileHashForPath(path) } } diff --git a/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala index e6055822ea9..b4d775f7389 100644 --- a/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/routes/wes/ServiceInfoSpec.scala @@ -24,7 +24,7 @@ class ServiceInfoSpec extends AsyncFlatSpec with ScalatestRouteTest with Matcher val expectedResponse = WesStatusInfoResponse(Map("CWL" -> Set("v1.0"), "WDL" -> Set("draft-2", "1.0", "biscayne")), List("1.0"), - Set("ftp", "s3", "drs", "gcs", "oss", "http"), + Set("ftp", "s3", "drs", "gcs", "http"), Map("Cromwell" -> CromwellApiService.cromwellVersion), List(), Map(WesState.Running -> 5, WesState.Queued -> 3, WesState.Canceling -> 2), diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala index bb9959ae29f..f22023c87fd 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala @@ -58,7 +58,7 @@ case class DrsPathBuilder(fileSystemProvider: DrsCloudNioFileSystemProvider, } val gcsPathOption = for { - // Right now, only pre-resolving GCS. In the future, could pull others like FTP, HTTP, S3, OSS, SRA, etc. + // Right now, only pre-resolving GCS. In the future, could pull others like FTP, HTTP, S3, SRA, etc. gsUriOption <- logAttempt( "resolve the uri through Martha", DrsResolver.getSimpleGsUri(pathAsString, fileSystemProvider.drsPathResolver).unsafeRunSync() diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/OssPathBuilder.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/OssPathBuilder.scala deleted file mode 100644 index 8792194f804..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/OssPathBuilder.scala +++ /dev/null @@ -1,157 +0,0 @@ -package cromwell.filesystems.oss - -import java.net.URI - -import com.google.common.net.UrlEscapers -import com.typesafe.config.Config -import net.ceedubs.ficus.Ficus._ -import cats.syntax.apply._ -import com.aliyun.oss.OSSClient -import common.validation.Validation._ -import cromwell.core.WorkflowOptions -import cromwell.core.path.{NioPath, Path, PathBuilder} -import cromwell.filesystems.oss.OssPathBuilder._ -import cromwell.filesystems.oss.nio._ - -import scala.language.postfixOps -import scala.util.matching.Regex -import scala.util.{Failure, Try} - -object OssPathBuilder { - - val URI_SCHEME = OssStorageFileSystem.URI_SCHEMA - - val OssBucketPattern:Regex = - """ - (?x) # Turn on comments and whitespace insensitivity - ^oss:// - ( # Begin capturing group for oss bucket name - [a-z0-9][a-z0-9-_\\.]+[a-z0-9] # Regex for bucket name - soft validation, see comment above - ) # End capturing group for gcs bucket name - (?: - /.* # No validation here - )? - """.trim.r - - sealed trait OssPathValidation - - case class ValidFullOssPath(bucket: String, path: String) extends OssPathValidation - - case object PossiblyValidRelativeOssPath extends OssPathValidation - - sealed trait InvalidOssPath extends OssPathValidation { - def pathString: String - def errorMessage: String - } - - final case class InvalidScheme(pathString: String) extends InvalidOssPath { - def errorMessage = s"OSS URIs must have 'oss' scheme: $pathString" - } - - final case class InvalidFullOssPath(pathString: String) extends InvalidOssPath { - def errorMessage = { - s""" - |The path '$pathString' does not seem to be a valid OSS path. - |Please check that it starts with oss:// and that the bucket and object follow OSS naming guidelines. - """.stripMargin.replaceAll("\n", " ").trim - } - } - - final case class UnparseableOssPath(pathString: String, throwable: Throwable) extends InvalidOssPath { - def errorMessage: String = - List(s"The specified OSS path '$pathString' does not parse as a URI.", throwable.getMessage).mkString("\n") - } - - private def softBucketParsing(string: String): Option[String] = string match { - case OssBucketPattern(bucket) => Option(bucket) - case _ => None - } - - def validateOssPath(string: String): OssPathValidation = { - Try { - val uri = URI.create(UrlEscapers.urlFragmentEscaper().escape(string)) - if (uri.getScheme == null) PossiblyValidRelativeOssPath - else if (uri.getScheme.equalsIgnoreCase(URI_SCHEME)) { - if (uri.getHost == null) { - softBucketParsing(string) map { ValidFullOssPath(_, uri.getPath) } getOrElse InvalidFullOssPath(string) - } else ValidFullOssPath(uri.getHost, uri.getPath) - } else InvalidScheme(string) - } recover { case t => UnparseableOssPath(string, t) } get - } - - def isOssPath(nioPath: NioPath): Boolean = { - nioPath.getFileSystem.provider().getScheme.equalsIgnoreCase(URI_SCHEME) - } - - def fromConfiguration(configuration: OssStorageConfiguration, - options: WorkflowOptions): OssPathBuilder = { - OssPathBuilder(configuration) - } - - def fromConfig(config: Config, options: WorkflowOptions): OssPathBuilder = { - val refresh = config.as[Option[Long]](TTLOssStorageConfiguration.RefreshInterval) - - val (endpoint, accessId, accessKey, securityToken) = ( - validate { config.as[String]("auth.endpoint") }, - validate { config.as[String]("auth.access-id") }, - validate { config.as[String]("auth.access-key") }, - validate { config.as[Option[String]]("auth.security-token") } - ).tupled.unsafe("OSS filesystem configuration is invalid") - - refresh match { - case None => - val cfg = DefaultOssStorageConfiguration(endpoint, accessId, accessKey, securityToken) - fromConfiguration(cfg, options) - case Some(_) => - val cfg = TTLOssStorageConfiguration(config) - fromConfiguration(cfg, options) - } - } -} - -final case class OssPathBuilder(ossStorageConfiguration: OssStorageConfiguration) extends PathBuilder { - def build(string: String): Try[OssPath] = { - validateOssPath(string) match { - case ValidFullOssPath(bucket, path) => - Try { - val ossStorageFileSystem = OssStorageFileSystem(bucket, ossStorageConfiguration) - OssPath(ossStorageFileSystem.getPath(path), ossStorageFileSystem.provider.ossClient) - } - case PossiblyValidRelativeOssPath => Failure(new IllegalArgumentException(s"$string does not have a oss scheme")) - case invalid: InvalidOssPath => Failure(new IllegalArgumentException(invalid.errorMessage)) - } - } - - override def name: String = "Object Storage Service" -} - -final case class BucketAndObj(bucket: String, obj: String) - -final case class OssPath private[oss](nioPath: NioPath, - ossClient: OSSClient) extends Path { - - override protected def newPath(path: NioPath): OssPath = { - OssPath(path, ossClient) - } - - override def pathAsString: String = ossStoragePath.pathAsString - - override def pathWithoutScheme: String = { - ossStoragePath.bucket + ossStoragePath.toAbsolutePath.toString - } - - def bucket: String = { - ossStoragePath.bucket - } - - def key: String = { - ossStoragePath.key - } - - lazy val eTag = ossClient.getSimplifiedObjectMeta(bucket, key).getETag - - def ossStoragePath: OssStoragePath = nioPath match { - case ossPath: OssStoragePath => ossPath - case _ => throw new RuntimeException(s"Internal path was not a cloud storage path: $nioPath") - } -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/OssPathBuilderFactory.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/OssPathBuilderFactory.scala deleted file mode 100644 index 666ba1c4849..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/OssPathBuilderFactory.scala +++ /dev/null @@ -1,14 +0,0 @@ -package cromwell.filesystems.oss - -import akka.actor.ActorSystem -import com.typesafe.config.Config -import cromwell.core.WorkflowOptions -import cromwell.core.path.PathBuilderFactory - -import scala.concurrent.{ExecutionContext, Future} - -final case class OssPathBuilderFactory(globalConfig: Config, instanceConfig: Config) extends PathBuilderFactory { - def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = { - Future.successful(OssPathBuilder.fromConfig(instanceConfig, options)) - } -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/batch/OssBatchCommandBuilder.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/batch/OssBatchCommandBuilder.scala deleted file mode 100644 index 8e1fc9a8135..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/batch/OssBatchCommandBuilder.scala +++ /dev/null @@ -1,35 +0,0 @@ -package cromwell.filesystems.oss.batch - -import cromwell.core.io._ -import cromwell.core.path.Path -import cromwell.filesystems.oss.OssPath - -import scala.util.Try - -private case object PartialOssBatchCommandBuilder extends PartialIoCommandBuilder { - override def sizeCommand: PartialFunction[Path, Try[IoSizeCommand]] = { - case ossPath: OssPath => Try(OssBatchSizeCommand(ossPath)) - } - - override def deleteCommand: PartialFunction[(Path, Boolean), Try[IoDeleteCommand]] = { - case (ossPath: OssPath, swallowIoExceptions) => Try(OssBatchDeleteCommand(ossPath, swallowIoExceptions)) - } - - override def copyCommand: PartialFunction[(Path, Path), Try[IoCopyCommand]] = { - case (ossSrc: OssPath, ossDest: OssPath) => Try(OssBatchCopyCommand(ossSrc, ossDest)) - } - - override def hashCommand: PartialFunction[Path, Try[IoHashCommand]] = { - case ossPath: OssPath => Try(OssBatchEtagCommand(ossPath)) - } - - override def touchCommand: PartialFunction[Path, Try[IoTouchCommand]] = { - case ossPath: OssPath => Try(OssBatchTouchCommand(ossPath)) - } - - override def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = { - case ossPath: OssPath => Try(OssBatchExistsCommand(ossPath)) - } -} - -case object OssBatchCommandBuilder extends IoCommandBuilder(List(PartialOssBatchCommandBuilder)) diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/batch/OssBatchIoCommand.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/batch/OssBatchIoCommand.scala deleted file mode 100644 index 625a068a126..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/batch/OssBatchIoCommand.scala +++ /dev/null @@ -1,98 +0,0 @@ -package cromwell.filesystems.oss.batch - -import com.aliyun.oss.OSSException -import com.aliyun.oss.model._ -import com.google.api.client.http.HttpHeaders -import cromwell.core.io._ -import cromwell.filesystems.oss._ - -/** - * Io commands with OSS paths and some logic enabling batching of request. - * @tparam T Return type of the IoCommand - * @tparam U Return type of the OSS response - */ -sealed trait OssBatchIoCommand[T, U] extends IoCommand[T] { - /** - * StorageRequest operation to be executed by this command - */ - def operation: Any - - /** - * Maps the Oss response of type U to the Cromwell Io response of type T - */ - protected def mapOssResponse(response: U): T - - /** - * Method called in the success callback of a batched request to decide what to do next. - * Returns an `Either[T, OssBatchIoCommand[T, U]]` - * Left(value) means the command is complete, and the result can be sent back to the sender. - * Right(newCommand) means the command is not complete and needs another request to be executed. - * Most commands will reply with Left(value). - */ - def onSuccess(response: U, httpHeaders: HttpHeaders): Either[T, OssBatchIoCommand[T, U]] = { - Left(mapOssResponse(response)) - } - - /** - * Override to handle a failure differently and potentially return a successful response. - */ - def onFailure(ossError: OSSException): Option[Either[T, OssBatchIoCommand[T, U]]] = None -} - -case class OssBatchCopyCommand( - override val source: OssPath, - override val destination: OssPath, - ) - extends IoCopyCommand(source, destination) with OssBatchIoCommand[Unit, CopyObjectResult] { - override def operation: GenericResult = { - val getObjectRequest = new CopyObjectRequest(source.bucket, source.key, destination.bucket, destination.key) - // TODO: Copy other attributes (encryption, metadata, etc.) - source.ossClient.copyObject(getObjectRequest) - } - override def mapOssResponse(response: CopyObjectResult): Unit = () - override def commandDescription: String = s"OssBatchCopyCommand source '$source' destination '$destination'" -} - -case class OssBatchDeleteCommand( - override val file: OssPath, - override val swallowIOExceptions: Boolean - ) extends IoDeleteCommand(file, swallowIOExceptions) with OssBatchIoCommand[Unit, Void] { - def operation: Unit = { - file.ossClient.deleteObject(file.bucket, file.key) - () - } - override protected def mapOssResponse(response: Void): Unit = () - override def commandDescription: String = s"OssBatchDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" -} - -/** - * Base trait for commands that use the headObject() operation. (e.g: size, crc32, ...) - */ -sealed trait OssBatchHeadCommand[T] extends OssBatchIoCommand[T, ObjectMetadata] { - def file: OssPath - - override def operation: ObjectMetadata = file.ossClient.getObjectMetadata(file.bucket, file.key) -} - -case class OssBatchSizeCommand(override val file: OssPath) extends IoSizeCommand(file) with OssBatchHeadCommand[Long] { - override def mapOssResponse(response: ObjectMetadata): Long = response.getContentLength - override def commandDescription: String = s"OssBatchSizeCommand file '$file'" -} - -case class OssBatchEtagCommand(override val file: OssPath) extends IoHashCommand(file) with OssBatchHeadCommand[String] { - override def mapOssResponse(response: ObjectMetadata): String = response.getETag - override def commandDescription: String = s"OssBatchEtagCommand file '$file'" -} - -case class OssBatchTouchCommand(override val file: OssPath) extends IoTouchCommand(file) with OssBatchHeadCommand[Unit] { - override def mapOssResponse(response: ObjectMetadata): Unit = () - override def commandDescription: String = s"OssBatchTouchCommand file '$file'" -} - -case class OssBatchExistsCommand(override val file: OssPath) extends IoExistsCommand(file) with OssBatchIoCommand[Boolean, Boolean] { - override def operation: Boolean = { - file.ossClient.doesObjectExist(file.bucket, file.key) - } - override def mapOssResponse(response: Boolean): Boolean = response - override def commandDescription: String = s"OssBatchExistsCommand file '$file'" -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssAppendOutputStream.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssAppendOutputStream.scala deleted file mode 100644 index bca8d16a609..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssAppendOutputStream.scala +++ /dev/null @@ -1,86 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.io.{ByteArrayInputStream, OutputStream} - -import com.aliyun.oss.OSSClient -import com.aliyun.oss.model.{AppendObjectRequest, GenericRequest} - -import scala.util.Try - -final case class OssAppendOutputStream(ossClient: OSSClient, path: OssStoragePath, deleteIfExists: Boolean) extends OutputStream { - - var position: Long = { - val exist = OssStorageRetry.fromTry( - () => Try{ - val request = new GenericRequest(path.bucket, path.key) - request.setLogEnabled(false) - ossClient.doesObjectExist(request) - } - ) - - var len: Long = 0 - if (exist && deleteIfExists) { - OssStorageRetry.from( - () => ossClient.deleteObject(path.bucket, path.key) - ) - } - else if (exist) { - len = OssStorageRetry.from( - () => ossClient.getObjectMetadata(path.bucket, path.key).getContentLength - ) - } - - len - } - - override def write(b: Int): Unit = { - val arr = Array[Byte]((b & 0xFF).toByte) - - val appendObjectRequest: AppendObjectRequest = new AppendObjectRequest(path.bucket, path.key, new ByteArrayInputStream(arr)) - this.synchronized { - appendObjectRequest.setPosition(position) - val appendObjectResult = OssStorageRetry.fromTry( - () => Try{ - ossClient.appendObject(appendObjectRequest) - } - ) - - position = appendObjectResult.getNextPosition() - } - } - - override def write(b: Array[Byte]): Unit = { - val appendObjectRequest: AppendObjectRequest = new AppendObjectRequest(path.bucket, path.key, new ByteArrayInputStream(b)) - this.synchronized { - appendObjectRequest.setPosition(position) - val appendObjectResult = OssStorageRetry.fromTry( - () => Try{ - ossClient.appendObject(appendObjectRequest) - } - ) - position = appendObjectResult.getNextPosition() - } - } - - override def write(b: Array[Byte], off: Int, len: Int): Unit = { - if (b == null) { - throw new NullPointerException - } else if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) { - throw new IndexOutOfBoundsException - } else if (len == 0) { - return - } - - val s = b.slice(off, off+len) - val appendObjectRequest: AppendObjectRequest = new AppendObjectRequest(path.bucket, path.key, new ByteArrayInputStream(s)) - this.synchronized { - appendObjectRequest.setPosition(position) - val appendObjectResult = OssStorageRetry.fromTry( - () => Try{ - ossClient.appendObject(appendObjectRequest) - } - ) - position = appendObjectResult.getNextPosition() - } - } -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssFileChannel.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssFileChannel.scala deleted file mode 100644 index e5851c9133e..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssFileChannel.scala +++ /dev/null @@ -1,19 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.ByteBuffer -import java.nio.channels.SeekableByteChannel - - -trait OssFileChannel extends SeekableByteChannel { - - override def isOpen: Boolean = true - - override def close(): Unit = {} - - override def read(dst: ByteBuffer): Int = throw new UnsupportedOperationException() - - override def write(src: ByteBuffer): Int = throw new UnsupportedOperationException() - - override def truncate(size: Long): SeekableByteChannel = throw new UnsupportedOperationException() - -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssFileReadChannel.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssFileReadChannel.scala deleted file mode 100644 index 1fb283b85e0..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssFileReadChannel.scala +++ /dev/null @@ -1,80 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.ByteBuffer -import java.nio.channels.{Channels, SeekableByteChannel} - -import com.aliyun.oss.OSSClient -import com.aliyun.oss.model.{GenericRequest, GetObjectRequest} - -import scala.util.Try - -final case class OssFileReadChannel(ossClient: OSSClient, pos: Long, path: OssStoragePath) extends OssFileChannel { - var internalPosition = pos - - override def position(): Long = { - synchronized { - internalPosition - } - } - - override def position(newPosition: Long): SeekableByteChannel = { - if (newPosition < 0) { - throw new IllegalArgumentException(newPosition.toString) - } - - synchronized { - if (newPosition != internalPosition) { - internalPosition = newPosition - } - - return this - } - } - - override def read(dst: ByteBuffer): Int = { - dst.mark() - - dst.reset() - - val want = dst.capacity - val begin: Long = position() - var end: Long = position + want - 1 - if (begin < 0 || end < 0 || begin > end) { - throw new IllegalArgumentException(s"being $begin or end $end invalid") - } - - if (begin >= size) { - return -1 - } - - if (end >= size()) { - end = size() - 1 - } - - val getObjectRequest = new GetObjectRequest(path.bucket, path.key) - getObjectRequest.setRange(begin, end) - - OssStorageRetry.fromTry( - () => Try{ - val ossObject = ossClient.getObject(getObjectRequest) - val in = ossObject.getObjectContent - val channel = Channels.newChannel(in) - - val amt = channel.read(dst) - channel.close() - internalPosition += amt - amt - } - ) - } - - override def size(): Long = { - OssStorageRetry.fromTry( - () => Try { - val request = new GenericRequest(path.bucket, path.key) - request.setLogEnabled(false) - ossClient.getSimplifiedObjectMeta(request).getSize - } - ) - } -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageDirectoryAttributes.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageDirectoryAttributes.scala deleted file mode 100644 index 84ab82deab9..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageDirectoryAttributes.scala +++ /dev/null @@ -1,38 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.file.attribute.FileTime - -import scala.collection.mutable.Map - -final case class OssStorageDirectoryAttributes(path: OssStoragePath) extends OssStorageFileAttributes { - override def creationTime(): FileTime = FileTime.fromMillis(0) - - override def lastAccessTime(): FileTime = FileTime.fromMillis(0) - - override def lastModifiedTime(): FileTime = creationTime() - - override def isRegularFile: Boolean = false - - override def isDirectory: Boolean = true - - override def isSymbolicLink: Boolean = false - - override def isOther: Boolean = false - - override def size(): Long = 0 - - override def fileKey(): AnyRef = path.pathAsString - - override def expires: FileTime = FileTime.fromMillis(0) - - override def cacheControl(): Option[String] = None - - override def contentDisposition: Option[String] = None - - override def contentEncoding: Option[String] = None - - override def etag: Option[String] = None - - override def userMeta: Map[String, String] = Map.empty[String, String] - -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributes.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributes.scala deleted file mode 100644 index ab53b4f8d40..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributes.scala +++ /dev/null @@ -1,19 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.file.attribute.{BasicFileAttributes, FileTime} - -import scala.collection.mutable.Map - -trait OssStorageFileAttributes extends BasicFileAttributes { - def cacheControl(): Option[String] - - def contentDisposition: Option[String] - - def contentEncoding: Option[String] - - def expires: FileTime - - def etag: Option[String] - - def userMeta: Map[String, String] -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributesView.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributesView.scala deleted file mode 100644 index cba65649a98..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributesView.scala +++ /dev/null @@ -1,45 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.file.NoSuchFileException -import java.nio.file.attribute.{BasicFileAttributeView, FileTime} -import java.util.Date - -import com.aliyun.oss.OSSClient -import com.aliyun.oss.model.{CopyObjectRequest, CopyObjectResult, GenericRequest} - -import scala.util.Try - -final case class OssStorageFileAttributesView(ossClient: OSSClient, path: OssStoragePath) extends BasicFileAttributeView { - override def name(): String = OssStorageFileSystem.URI_SCHEMA - - override def readAttributes(): OssStorageFileAttributes = { - val ossPath = OssStoragePath.checkPath(path) - - if (ossPath.seemsLikeDirectory) { - return OssStorageDirectoryAttributes(path) - } - - val request = new GenericRequest(ossPath.bucket, ossPath.key) - request.setLogEnabled(false) - if (!ossClient.doesObjectExist(request)) { - throw new NoSuchFileException(path.toString) - } - - val objectMeta = OssStorageRetry.fromTry( - () => Try{ - ossClient.getObjectMetadata(path.bucket, path.key) - } - ) - - OssStorageObjectAttributes(objectMeta, path) - } - - override def setTimes(lastModifiedTime: FileTime, lastAccessTime: FileTime, createTime: FileTime): Unit = { - val meta = ossClient.getObjectMetadata(path.bucket, path.key) - meta.setLastModified(new Date(lastModifiedTime.toMillis)) - - val copyReq = new CopyObjectRequest(path.bucket, path.key, path.bucket, path.key) - copyReq.setNewObjectMetadata(meta) - val _: CopyObjectResult = ossClient.copyObject(copyReq) - } -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileSystem.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileSystem.scala deleted file mode 100644 index 3d27cecf296..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileSystem.scala +++ /dev/null @@ -1,161 +0,0 @@ -package cromwell.filesystems.oss.nio - - -import java.nio.file._ -import java.nio.file.attribute.UserPrincipalLookupService -import java.util.Objects -import java.{lang, util} - -import com.aliyun.oss.common.auth.DefaultCredentialProvider -import com.aliyun.oss.{ClientConfiguration, OSSClient} -import cromwell.filesystems.oss.nio.OssStorageConfiguration.{ACCESS_ID_KEY, ACCESS_KEY_KEY, ENDPOINT_KEY, SECURITY_TOKEN_KEY} - -import scala.jdk.CollectionConverters._ - - -object OssStorageFileSystem { - val SEPARATOR: String = "/" - val URI_SCHEMA: String = "oss" - val OSS_VIEW = "oss" - val BASIC_VIEW = "basic" - - def apply(provider: OssStorageFileSystemProvider, bucket: String, config: OssStorageConfiguration): OssStorageFileSystem = { - val res = new OssStorageFileSystem(bucket, config) - res.internalProvider = provider - res - } -} - -object OssStorageConfiguration { - val ENDPOINT_KEY = "endpoint" - val ACCESS_ID_KEY = "access-id" - val ACCESS_KEY_KEY = "access-key" - val SECURITY_TOKEN_KEY = "security-token" - - import scala.collection.immutable.Map - def parseMap(map: Map[String, Any]): OssStorageConfiguration = { - val endpoint = map.get(ENDPOINT_KEY) match { - case Some(endpoint: String) if !endpoint.isEmpty => endpoint - case _ => throw new IllegalArgumentException(s"endpoint is mandatory and must be an unempty string") - } - val accessId = map.get(ACCESS_ID_KEY) match { - case Some(id: String) if !id.isEmpty => id - case _ => throw new IllegalArgumentException(s"access-id is mandatory and must be an unempty string") - } - - val accessKey = map.get(ACCESS_KEY_KEY) match { - case Some(key: String) if !key.isEmpty => key - case _ => throw new IllegalArgumentException(s"access-key is mandatory and must be an unempty string") - } - - val securityToken = map.get(SECURITY_TOKEN_KEY) match { - case Some(token: String) if !token.isEmpty => Some(token) - case _ => None - } - - new DefaultOssStorageConfiguration(endpoint, accessId, accessKey, securityToken) - } - - def getClient(map: Map[String, String]): OSSClient = { - parseMap(map).newOssClient() - } - - def getClient(endpoint: String, - accessId: String, - accessKey: String, - stsToken: Option[String]): OSSClient = { - DefaultOssStorageConfiguration(endpoint, accessId, accessKey, stsToken).newOssClient() - } - -} - -trait OssStorageConfiguration { - def endpoint: String - - def accessId: String - - def accessKey: String - - def securityToken: Option[String] - - def toMap: Map[String, String] = { - val ret = Map(ENDPOINT_KEY -> endpoint, ACCESS_ID_KEY -> accessId, ACCESS_KEY_KEY -> accessKey) - val token = securityToken map {token => SECURITY_TOKEN_KEY -> token} - ret ++ token - } - - def newOssClient() = { - val credentialsProvider = securityToken match { - case Some(token: String) => - new DefaultCredentialProvider(accessId, accessKey, token) - case None => - new DefaultCredentialProvider(accessId, accessKey) - } - val clientConfiguration = new ClientConfiguration - new OSSClient(endpoint, credentialsProvider, clientConfiguration) - } - -} - -case class DefaultOssStorageConfiguration(endpoint: String, accessId: String, accessKey: String, securityToken: Option[String] = None) extends OssStorageConfiguration {} - -case class OssStorageFileSystem(bucket: String, config: OssStorageConfiguration) extends FileSystem { - - var internalProvider: OssStorageFileSystemProvider = OssStorageFileSystemProvider(config) - - override def provider: OssStorageFileSystemProvider = internalProvider - - override def getPath(first: String, more: String*): OssStoragePath = OssStoragePath.getPath(this, first, more: _*) - - override def close(): Unit = { - // do nothing currently. - } - - override def isOpen: Boolean = { - true - } - - override def isReadOnly: Boolean = { - false - } - - override def getSeparator: String = { - OssStorageFileSystem.SEPARATOR - } - - override def getRootDirectories: lang.Iterable[Path] = { - Set[Path](OssStoragePath.getPath(this, UnixPath.ROOT_PATH)).asJava - } - - override def getFileStores: lang.Iterable[FileStore] = { - Set.empty[FileStore].asJava - } - - override def getPathMatcher(syntaxAndPattern: String): PathMatcher = { - FileSystems.getDefault.getPathMatcher(syntaxAndPattern) - } - - override def getUserPrincipalLookupService: UserPrincipalLookupService = { - throw new UnsupportedOperationException() - } - - override def newWatchService(): WatchService = { - throw new UnsupportedOperationException() - } - - override def supportedFileAttributeViews(): util.Set[String] = { - Set(OssStorageFileSystem.OSS_VIEW, OssStorageFileSystem.BASIC_VIEW).asJava - } - - override def equals(obj: scala.Any): Boolean = { - this == obj || - obj.isInstanceOf[OssStorageFileSystem] && - obj.asInstanceOf[OssStorageFileSystem].config.equals(config) && - obj.asInstanceOf[OssStorageFileSystem].bucket.equals(bucket) - } - - override def hashCode(): Int = Objects.hash(bucket) - - override def toString: String = OssStorageFileSystem.URI_SCHEMA + "://" + bucket -} - diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemProvider.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemProvider.scala deleted file mode 100644 index c820e66f411..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemProvider.scala +++ /dev/null @@ -1,323 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.io.{BufferedOutputStream, OutputStream} -import java.net.URI -import java.nio.channels.SeekableByteChannel -import java.nio.file._ -import java.nio.file.attribute.{BasicFileAttributeView, BasicFileAttributes, FileAttribute, FileAttributeView} -import java.nio.file.spi.FileSystemProvider -import java.util - -import com.aliyun.oss.OSSClient -import com.aliyun.oss.model.{GenericRequest, ListObjectsRequest} -import com.google.common.collect.AbstractIterator - -import scala.jdk.CollectionConverters._ -import scala.collection.immutable.Set -import collection.mutable.ArrayBuffer - - -final case class OssStorageFileSystemProvider(config: OssStorageConfiguration) extends FileSystemProvider { - def ossClient: OSSClient = config.newOssClient() - - class PathIterator(ossClient: OSSClient, prefix: OssStoragePath, filter: DirectoryStream.Filter[_ >: Path]) extends AbstractIterator[Path] { - var nextMarker: Option[String] = None - - var iterator: Iterator[String] = Iterator() - - override def computeNext(): Path = { - if (!iterator.hasNext) { - nextMarker match { - case None => iterator = listNext("") - case Some(marker: String) if !marker.isEmpty => iterator = listNext(marker) - case Some(marker: String) if marker.isEmpty => iterator = Iterator() - case Some(null) => iterator = Iterator() - case oh => throw new RuntimeException(s"Programmer Error! Unexpected case match: $oh") - } - } - - - if (iterator.hasNext) { - val path = OssStoragePath.getPath(prefix.getFileSystem, iterator.next()) - if (filter.accept(path)) { - path - } else { - computeNext() - } - } else { - endOfData() - } - } - - private[this] def listNext(marker: String): Iterator[String] = { - val objectListing = OssStorageRetry.from( - () => { - val listObjectRequest = new ListObjectsRequest(prefix.bucket) - listObjectRequest.setDelimiter(UnixPath.SEPARATOR.toString) - listObjectRequest.setPrefix(prefix.key) - listObjectRequest.setMarker(marker) - - ossClient.listObjects(listObjectRequest) - } - ) - - val result = ArrayBuffer.empty[String] - - objectListing.getObjectSummaries.asScala.filterNot(_.equals(prefix.key)).foreach(obj => {result append obj.getKey.stripPrefix(prefix.key)}) - objectListing.getCommonPrefixes.asScala.filterNot(_.equals(prefix.key)).foreach(obj => {result append obj.stripPrefix(prefix.key)}) - - nextMarker = Some(objectListing.getNextMarker) - result.iterator - } - } - - class OssStorageDirectoryStream(ossClient: OSSClient, prefix: OssStoragePath, filter: DirectoryStream.Filter[_ >: Path]) extends DirectoryStream[Path] { - - override def iterator(): util.Iterator[Path] = new PathIterator(ossClient, prefix, filter) - - override def close(): Unit = {} - - } - - override def getScheme: String = OssStorageFileSystem.URI_SCHEMA - - override def newFileSystem(uri: URI, env: util.Map[String, _]): OssStorageFileSystem = { - if (uri.getScheme != getScheme) { - throw new IllegalArgumentException(s"Schema ${uri.getScheme} not match") - } - - val bucket = uri.getHost - if (bucket.isEmpty) { - throw new IllegalArgumentException(s"Bucket is empty") - } - - if (uri.getPort != -1) { - throw new IllegalArgumentException(s"Port is not permitted") - } - - OssStorageFileSystem(this, bucket, OssStorageConfiguration.parseMap(env.asScala.toMap)) - } - - override def getFileSystem(uri: URI): OssStorageFileSystem = { - newFileSystem(uri, config.toMap.asJava) - } - - override def getPath(uri: URI): OssStoragePath = { - OssStoragePath.getPath(getFileSystem(uri), uri.getPath) - } - - override def newOutputStream(path: Path, options: OpenOption*): OutputStream = { - if (!path.isInstanceOf[OssStoragePath]) { - throw new ProviderMismatchException(s"Not a oss storage path $path") - } - - val len = options.length - - var opts = Set[OpenOption]() - if (len == 0) { - opts = Set(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) - } else { - for (opt <- options) { - if (opt == StandardOpenOption.READ) { - throw new IllegalArgumentException("READ not allowed") - } - - opts += opt - } - } - - opts += StandardOpenOption.WRITE - val ossStream = OssAppendOutputStream(ossClient, path.asInstanceOf[OssStoragePath], true) - - new BufferedOutputStream(ossStream, 256*1024) - } - - override def newByteChannel(path: Path, options: util.Set[_ <: OpenOption], attrs: FileAttribute[_]*): SeekableByteChannel = { - if (!path.isInstanceOf[OssStoragePath]) { - throw new ProviderMismatchException(s"Not a oss storage path $path") - } - - for (opt <- options.asScala) { - opt match { - case StandardOpenOption.READ => - case StandardOpenOption.WRITE => throw new IllegalArgumentException(s"WRITE byte channel not allowed currently, $path") - case StandardOpenOption.SPARSE | StandardOpenOption.TRUNCATE_EXISTING => - case StandardOpenOption.APPEND | StandardOpenOption.CREATE | StandardOpenOption.DELETE_ON_CLOSE | - StandardOpenOption.CREATE_NEW | StandardOpenOption.DSYNC | StandardOpenOption.SYNC => throw new UnsupportedOperationException() - } - } - - OssFileReadChannel(ossClient, 0, path.asInstanceOf[OssStoragePath]) - } - - def doesObjectExist(bucket: String, name: String): Boolean = { - val req = new GenericRequest(bucket, name) - req.setLogEnabled(false) - ossClient.doesBucketExist(req) - } - - override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = {} - - override def deleteIfExists(path: Path): Boolean = { - val ossPath = OssStoragePath.checkPath(path) - - if (ossPath.seemsLikeDirectory) { - if (headPrefix(ossPath)) { - throw new UnsupportedOperationException("Can not delete a non-empty directory") - } - - return true - } - - val exist = OssStorageRetry.from( - () => { - val request = new GenericRequest(ossPath.bucket, ossPath.key) - request.setLogEnabled(false) - ossClient.doesObjectExist(request) - } - ) - - if (!exist) { - return false - } - - OssStorageRetry.from( - () => ossClient.deleteObject(ossPath.bucket, ossPath.key) - ) - - true - } - - override def delete(path: Path): Unit = { - if (!deleteIfExists(path)) { - throw new NoSuchFileException(s"File $path not exists") - } - } - - /* - * XXX: Can only copy files whose size is below 1GB currently. - */ - - override def copy(source: Path, target: Path, options: CopyOption*): Unit = { - val srcOssPath = OssStoragePath.checkPath(source) - val targetOssPath= OssStoragePath.checkPath(target) - - // ignore all options currently. - if (srcOssPath == targetOssPath) { - return - } - - val _ = OssStorageRetry.from( - () => ossClient.copyObject(srcOssPath.bucket, srcOssPath.key, targetOssPath.bucket, targetOssPath.key) - ) - - } - - override def move(source: Path, target: Path, options: CopyOption*): Unit = { - copy(source, target, options: _*) - - val _ = deleteIfExists(source) - } - - override def isSameFile(path: Path, path2: Path): Boolean = { - OssStoragePath.checkPath(path).equals(OssStoragePath.checkPath(path2)) - } - - override def isHidden(path: Path): Boolean = { - false - } - - override def getFileStore(path: Path): FileStore = throw new UnsupportedOperationException() - - override def checkAccess(path: Path, modes: AccessMode*): Unit = { - for (mode <- modes) { - mode match { - case AccessMode.READ | AccessMode.WRITE => - case AccessMode.EXECUTE => throw new AccessDeniedException(mode.toString) - } - } - - val ossPath = OssStoragePath.checkPath(path) - // directory always exists. - if (ossPath.seemsLikeDirectory) { - return - } - - val exist = OssStorageRetry.from( - () => { - val request = new GenericRequest(ossPath.bucket, ossPath.key) - request.setLogEnabled(false) - ossClient.doesObjectExist(request) - } - ) - - if (!exist) { - throw new NoSuchFileException(path.toString) - } - } - - override def getFileAttributeView[V <: FileAttributeView](path: Path, `type`: Class[V], options: LinkOption*): V = { - if (`type` != classOf[OssStorageFileAttributesView] && `type` != classOf[BasicFileAttributeView] ) { - throw new UnsupportedOperationException(`type`.getSimpleName) - } - - val ossPath = OssStoragePath.checkPath(path) - - OssStorageFileAttributesView(ossClient, ossPath).asInstanceOf[V] - } - - override def readAttributes(path: Path, attributes: String, options: LinkOption*): util.Map[String, AnyRef] = { - throw new UnsupportedOperationException() - } - - override def readAttributes[A <: BasicFileAttributes](path: Path, `type`: Class[A], options: LinkOption*): A = { - if (`type` != classOf[OssStorageFileAttributes] && `type` != classOf[BasicFileAttributes] ) { - throw new UnsupportedOperationException(`type`.getSimpleName) - } - - val ossPath = OssStoragePath.checkPath(path) - - if (ossPath.seemsLikeDirectory) { - return new OssStorageDirectoryAttributes(ossPath).asInstanceOf[A] - } - - val exists = OssStorageRetry.from( - () => { - val request = new GenericRequest(ossPath.bucket, ossPath.key) - request.setLogEnabled(false) - ossClient.doesObjectExist(request) - } - ) - - if (!exists) { - throw new NoSuchFileException(ossPath.toString) - } - - val objectMeta = OssStorageRetry.from( - () => ossClient.getObjectMetadata(ossPath.bucket, ossPath.key) - ) - - OssStorageObjectAttributes(objectMeta, ossPath).asInstanceOf[A] - } - - override def newDirectoryStream(dir: Path, filter: DirectoryStream.Filter[_ >: Path]): DirectoryStream[Path] = { - val ossPath = OssStoragePath.checkPath(dir) - - new OssStorageDirectoryStream(ossClient, ossPath, filter) - } - - override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = throw new UnsupportedOperationException() - - private[this] def headPrefix(path: Path): Boolean = { - val ossPath = OssStoragePath.checkPath(path) - - val listRequest = new ListObjectsRequest(ossPath.bucket) - listRequest.setPrefix(ossPath.key) - - val listResult = OssStorageRetry.from( - () => ossClient.listObjects(listRequest) - ) - - listResult.getObjectSummaries.iterator().hasNext - } -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageObjectAttributes.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageObjectAttributes.scala deleted file mode 100644 index 376aca7f2f8..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageObjectAttributes.scala +++ /dev/null @@ -1,44 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.file.attribute.FileTime - -import com.aliyun.oss.model.ObjectMetadata - -import scala.jdk.CollectionConverters._ -import scala.collection.mutable.Map -import scala.util.Try - -final case class OssStorageObjectAttributes(objectMeta: ObjectMetadata, path: OssStoragePath) extends OssStorageFileAttributes { - override def creationTime(): FileTime = { - FileTime.fromMillis(objectMeta.getLastModified.getTime) - } - - override def lastAccessTime(): FileTime = FileTime.fromMillis(0) - - override def lastModifiedTime(): FileTime = creationTime() - - override def isRegularFile: Boolean = true - - override def isDirectory: Boolean = false - - override def isSymbolicLink: Boolean = false - - override def isOther: Boolean = false - - override def size(): Long = objectMeta.getContentLength - - override def fileKey(): AnyRef = path.pathAsString - - // oss sdk has an issule: throw NullPointerException when no expire time exists. - override def expires: FileTime = FileTime.fromMillis(Try{objectMeta.getExpirationTime.getTime} getOrElse (0)) - - override def cacheControl(): Option[String] = Option(objectMeta.getCacheControl) - - override def contentDisposition: Option[String] = Option(objectMeta.getContentDisposition) - - override def contentEncoding: Option[String] = Option(objectMeta.getContentEncoding) - - override def etag: Option[String] = Option(objectMeta.getETag) - - override def userMeta: Map[String, String] = objectMeta.getUserMetadata.asScala -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStoragePath.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStoragePath.scala deleted file mode 100644 index 02cd06b090e..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStoragePath.scala +++ /dev/null @@ -1,252 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.io.File -import java.net.URI -import java.nio.file._ -import java.util -import java.util.Objects - -import com.google.common.collect.UnmodifiableIterator - -object OssStoragePath { - def checkOssStoragePath(other: Path): OssStoragePath = { - if (!other.isInstanceOf[OssStoragePath]) { - throw new ProviderMismatchException(s"Not a oss storage path $other") - } - - other.asInstanceOf[OssStoragePath] - } - - def getPath(filesystem: OssStorageFileSystem, path: UnixPath) = new OssStoragePathImpl(filesystem, path) - - def getPath(filesystem: OssStorageFileSystem, first: String, more: String*) = new OssStoragePathImpl(filesystem, UnixPath.getPath(first, more: _*)) - - def checkPath(path: Path): OssStoragePath = { - if (!path.isInstanceOf[OssStoragePath]) { - throw new ProviderMismatchException(s"Not an oss storage path $path") - } - - path.asInstanceOf[OssStoragePath] - } - -} - -trait OssStoragePath extends Path { - def bucket = "" - - def key = "" - - def path: UnixPath = UnixPath.EMPTY_PATH - - def seemsLikeDirectory = false - - def pathAsString: String = "" - - override def getFileSystem: OssStorageFileSystem = throw new UnsupportedOperationException - - override def isAbsolute: Boolean = throw new UnsupportedOperationException - - override def getRoot: OssStoragePath = throw new UnsupportedOperationException - - override def getFileName: OssStoragePath = throw new UnsupportedOperationException - - override def getParent: OssStoragePath = throw new UnsupportedOperationException - - override def getNameCount: Int = throw new UnsupportedOperationException - - override def getName(index: Int): OssStoragePath = throw new UnsupportedOperationException - - override def subpath(beginIndex: Int, endIndex: Int): OssStoragePath = throw new UnsupportedOperationException - - override def startsWith(other: Path): Boolean = throw new UnsupportedOperationException - - override def startsWith(other: String): Boolean = throw new UnsupportedOperationException - - override def endsWith(other: Path): Boolean = throw new UnsupportedOperationException - - override def endsWith(other: String): Boolean = throw new UnsupportedOperationException - - override def normalize(): OssStoragePath = throw new UnsupportedOperationException - - override def resolve(other: Path): OssStoragePath = throw new UnsupportedOperationException - - override def resolve(other: String): OssStoragePath = throw new UnsupportedOperationException - - override def resolveSibling(other: Path): OssStoragePath = throw new UnsupportedOperationException - - override def resolveSibling(other: String): OssStoragePath = throw new UnsupportedOperationException - - override def relativize(other: Path): OssStoragePath = throw new UnsupportedOperationException - - override def toAbsolutePath: OssStoragePath = throw new UnsupportedOperationException - - override def toRealPath(options: LinkOption*): OssStoragePath = throw new UnsupportedOperationException - - override def toFile: File = throw new UnsupportedOperationException - - override def register(watcher: WatchService, events: WatchEvent.Kind[_]*): WatchKey = throw new UnsupportedOperationException - - override def register(watcher: WatchService, events: Array[WatchEvent.Kind[_]], modifiers: WatchEvent.Modifier*): WatchKey = throw new UnsupportedOperationException - - override def iterator(): util.Iterator[Path] = throw new UnsupportedOperationException - - override def compareTo(other: Path): Int = throw new UnsupportedOperationException - - override def toUri: URI = throw new UnsupportedOperationException -} - -final case class OssStoragePathImpl(filesystem: OssStorageFileSystem, override val path: UnixPath = UnixPath.EMPTY_PATH) extends OssStoragePath { - - override def pathAsString: String = toUri.toString - - override def bucket: String = filesystem.bucket - - override def key: String = toAbsolutePath.toString.stripPrefix("/") - - override def getFileSystem: OssStorageFileSystem = filesystem - - override def isAbsolute: Boolean = path.isAbsolute - - override def getRoot: OssStoragePath = path.getRoot map {path => newPath(path)} getOrElse NullOssStoragePath(filesystem) - - override def getFileName: OssStoragePath = path.getFileName map {path => newPath(path)} getOrElse NullOssStoragePath(filesystem) - - override def getParent: OssStoragePath = path.getParent map {path => newPath(path)} getOrElse NullOssStoragePath(filesystem) - - override def getNameCount: Int = path.getNameCount - - override def getName(index: Int): OssStoragePath = path.getName(index) map {path => newPath(path)} getOrElse NullOssStoragePath(filesystem) - - override def subpath(beginIndex: Int, endIndex: Int): OssStoragePath = path.subPath(beginIndex, endIndex) map {path => newPath(path)} getOrElse NullOssStoragePath(filesystem) - - override def startsWith(other: Path): Boolean = { - if (!other.isInstanceOf[OssStoragePath]) { - return false - } - - val that = other.asInstanceOf[OssStoragePath] - if (bucket != that.bucket) { - return false - } - - path.startsWith(that.path) - } - - override def startsWith(other: String): Boolean = { - path.startsWith(UnixPath.getPath(other)) - } - - override def endsWith(other: Path): Boolean = { - if (!other.isInstanceOf[OssStoragePath]) { - return false - } - val that = other.asInstanceOf[OssStoragePath] - if (bucket != that.bucket) { - return false - } - - path.endsWith(that.path) - } - - override def endsWith(other: String): Boolean = { - path.endsWith(UnixPath.getPath(other)) - } - - override def normalize(): OssStoragePath = newPath(path.normalize()) - - override def resolve(other: Path): OssStoragePath = { - val that = OssStoragePath.checkOssStoragePath(other) - - newPath(path.resolve(that.path)) - } - - override def resolve(other: String): OssStoragePath = { - newPath(path.resolve(UnixPath.getPath(other))) - } - - override def resolveSibling(other: Path): OssStoragePath = { - val that = OssStoragePath.checkOssStoragePath(other) - - newPath(path.resolveSibling(that.path)) - } - - override def resolveSibling(other: String): OssStoragePath = { - newPath(path.resolveSibling(UnixPath.getPath(other))) - } - - override def relativize(other: Path): OssStoragePath = { - val that = OssStoragePath.checkOssStoragePath(other) - - newPath(path.relativize(that.path)) - } - - /** - * currently a mocked one - */ - override def toAbsolutePath: OssStoragePath = { - newPath(path.toAbsolutePath()) - } - - override def toRealPath(options: LinkOption*): OssStoragePath = toAbsolutePath - - override def toFile: File = throw new UnsupportedOperationException - - override def register(watcher: WatchService, events: WatchEvent.Kind[_]*): WatchKey = throw new UnsupportedOperationException - - override def register(watcher: WatchService, events: Array[WatchEvent.Kind[_]], modifiers: WatchEvent.Modifier*): WatchKey = throw new UnsupportedOperationException - - override def iterator(): util.Iterator[Path] = { - if (path.isEmpty() || path.isRoot) { - return util.Collections.emptyIterator() - } - - new PathIterator() - } - - override def compareTo(other: Path): Int = { - if (other.isInstanceOf[OssStoragePath]) { - return -1 - } - - val that = other.asInstanceOf[OssStoragePath] - val res: Int = bucket.compareTo(that.bucket) - if (res != 0) { - return res - } - - path.compareTo(that.path) - } - - override def seemsLikeDirectory = path.seemsLikeDirectory() - - override def equals(obj: scala.Any): Boolean = { - (this eq obj.asInstanceOf[AnyRef]) || obj.isInstanceOf[OssStoragePath] && obj.asInstanceOf[OssStoragePath].bucket.equals(bucket) && obj.asInstanceOf[OssStoragePath].path.equals(path) - } - - override def hashCode(): Int = { - Objects.hash(bucket, toAbsolutePath.toString) - } - - override def toString: String = path.toString - - override def toUri: URI = new URI("oss", bucket, toAbsolutePath.toString, None.orNull) - - private[this] def newPath(unixPath: UnixPath): OssStoragePath = { - if (unixPath == path) { - this - } else { - OssStoragePathImpl(filesystem, unixPath) - } - } - - - class PathIterator extends UnmodifiableIterator[Path] { - val delegate = path.split() - - override def next(): OssStoragePath = newPath(UnixPath.getPath(delegate.next())) - - override def hasNext: Boolean = delegate.hasNext - } -} - -final case class NullOssStoragePath(filesystem: OssStorageFileSystem) extends OssStoragePath diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageRetry.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageRetry.scala deleted file mode 100644 index 105c5458b5d..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/OssStorageRetry.scala +++ /dev/null @@ -1,71 +0,0 @@ -package cromwell.filesystems.oss.nio - -import com.aliyun.oss.{ClientException, OSSException} -import common.util.Backoff -import cromwell.core.retry.SimpleExponentialBackoff - -import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} - -object OssStorageRetry { - def fromTry[A](f: () => Try[A], - maxRetries: Option[Int] = Some(DEFAULT_MAX_RETRIES), - backoff: Backoff = SimpleExponentialBackoff(1.second, 60.seconds, 1.1), - isTransient: Throwable => Boolean = transient, - isFatal: Throwable => Boolean = fatal - ): A = { - val delay = backoff.backoffMillis - - f() match { - case Success(ret) => ret - case Failure(e) if isFatal(e) => throw e - case Failure(e) if !isFatal(e) => - val retriesLeft = if (isTransient(e)) maxRetries else maxRetries map { _ - 1 } - if (retriesLeft.forall(_ > 0)) { - Thread.sleep(delay) - fromTry(f, retriesLeft, backoff, isTransient, isFatal) - } else { - throw e - } - case oh => throw new RuntimeException(s"Programmer Error! Unexpected case match: $oh") - } - } - - def from[A](f: () => A, - maxRetries: Option[Int] = Some(DEFAULT_MAX_RETRIES), - backoff: Backoff = SimpleExponentialBackoff(1.second, 60.seconds, 1.1), - isTransient: Throwable => Boolean = transient, - isFatal: Throwable => Boolean = fatal - ): A = { - fromTry[A]( - () => Try{ - f() - }, - maxRetries, - backoff, - isTransient, - isFatal - ) - } - - def transient(t: Throwable): Boolean = t match { - case oss: OSSException if TRANSIENT_ERROR_CODES contains oss.getErrorCode => true - case _ => false - } - - def fatal(t: Throwable): Boolean = t match { - case oss: OSSException if oss.getErrorCode.startsWith("Invalid") || oss.getErrorCode.startsWith("NoSuch") => true - case _: OSSException | _: ClientException => false - case _ => true - } - - val DEFAULT_MAX_RETRIES = 10 - - val TRANSIENT_ERROR_CODES = Array("InternalError", - "RequestTimeout", - "RecvFlowLimitExceeded", - "SendFlowLimitExceeded", - "UploadTrafficRateLimitExceeded", - "DownloadTrafficRateLimitExceeded" - ) -} diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/TTLOssStorageConfiguration.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/TTLOssStorageConfiguration.scala deleted file mode 100644 index e87f59c25bb..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/TTLOssStorageConfiguration.scala +++ /dev/null @@ -1,47 +0,0 @@ -package cromwell.filesystems.oss.nio - -import com.aliyun.oss.OSSClient -import com.typesafe.config.Config -import net.ceedubs.ficus.Ficus._ - -object TTLOssStorageConfiguration { - def currentTimestamp = System.currentTimeMillis / 1000 - - def defaultRefreshInterval: Long = 30 * 60 - - val RefreshInterval = "refresh-interval" - - def apply(config: Config): TTLOssStorageConfiguration = new TTLOssStorageConfiguration(config) -} - -/* Unsupported. For test purposes only. */ -class TTLOssStorageConfiguration(config: Config) extends OssStorageConfiguration { - - override def endpoint: String = config.as[Option[String]](authPath(OssStorageConfiguration.ENDPOINT_KEY)) getOrElse("") - - override def accessId: String = config.as[Option[String]](authPath(OssStorageConfiguration.ACCESS_ID_KEY)) getOrElse("") - - override def accessKey: String = config.as[Option[String]](authPath(OssStorageConfiguration.ACCESS_KEY_KEY)) getOrElse("") - - override def securityToken: Option[String] = config.as[Option[String]](authPath(OssStorageConfiguration.SECURITY_TOKEN_KEY)) - - def refreshInterval: Long = config.as[Option[Long]](TTLOssStorageConfiguration.RefreshInterval).getOrElse(TTLOssStorageConfiguration.defaultRefreshInterval) - - private def authPath(key: String): String = s"auth.$key" - private var lastClientUpdateTime: Long = 0 - - private var oldClient: Option[OSSClient] = None - - override def newOssClient(): OSSClient = { - val current = TTLOssStorageConfiguration.currentTimestamp - synchronized { - if (lastClientUpdateTime == 0 || current - lastClientUpdateTime > refreshInterval) { - oldClient = Option(super.newOssClient()) - lastClientUpdateTime = current - } - } - - oldClient getOrElse(throw new IllegalArgumentException("Non oss client")) - } -} - diff --git a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/UnixPath.scala b/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/UnixPath.scala deleted file mode 100644 index fc640680a26..00000000000 --- a/filesystems/oss/src/main/scala/cromwell/filesystems/oss/nio/UnixPath.scala +++ /dev/null @@ -1,347 +0,0 @@ -package cromwell.filesystems.oss.nio - - -import scala.collection.mutable -import scala.math.Ordering.Implicits._ -import scala.util.control.Breaks._ -import scala.util.{Failure, Success, Try} - - -object UnixPath { - val DOT: Char = '.' - val SEPARATOR: Char = '/' - val ROOT: String = SEPARATOR.toString - val CURRENT_DIR: String = DOT.toString - val PARENT_DIR: String = DOT.toString + DOT.toString - val EMPTY_PATH: UnixPath = new UnixPath("") - val ROOT_PATH: UnixPath = new UnixPath("/") - - private def isRoot(path: String) = path.length() == 1 && path.charAt(0) == SEPARATOR - private def isAbsolute(path: String) = !path.isEmpty() && path.charAt(0) == SEPARATOR - private def hasTrailingSeparator(path: String) = !path.isEmpty() && path.charAt(path.length - 1) == SEPARATOR - - def getPath(path: String): UnixPath = { - if (path.isEmpty()) { - return EMPTY_PATH - } else if (isRoot(path)) { - return ROOT_PATH - } else { - UnixPath(path) - } - } - - def getPath(first: String, more: String*): UnixPath = { - if (more.length == 0) { - return new UnixPath(first) - } - - val builder = new StringBuilder(first) - for ((part, index) <- more.view.zipWithIndex) { - if (part.isEmpty()) { - // do nothing - } else if (isAbsolute(part)) { - if (index == more.length - 1) { - return new UnixPath(part) - } else { - builder.replace(0, builder.length, part) - } - } else if (hasTrailingSeparator(part)) { - builder.append(part) - } else { - builder.append(SEPARATOR) - builder.append(part) - } - } - - UnixPath(builder.toString) - } - -} - -final case class UnixPath(path: String) extends CharSequence -{ - lazy val parts = initParts() - - def isRoot = UnixPath.isRoot(path) - - def isAbsolute = UnixPath.isAbsolute(path) - - def isEmpty() = path.isEmpty() - - def hasTrailingSeparator = UnixPath.hasTrailingSeparator(path) - - def seemsLikeDirectory() = path.isEmpty() || hasTrailingSeparator || path.endsWith(".") && (length == 1 || path.charAt(length - 2) == UnixPath.SEPARATOR) || path.endsWith("..") && (length == 2 || path.charAt(length - 3) == UnixPath.SEPARATOR) - - def getFileName: Option[UnixPath] = { - if (path.isEmpty() || isRoot) { - None - } else { - if (parts.size == 1 && parts.last == path) { - Some(this) - } else { - Some(UnixPath(parts.last)) - } - } - } - - def getParent: Option[UnixPath] = { - if (path.isEmpty() || isRoot) { - return None - } - - val index = if (hasTrailingSeparator) path.lastIndexOf(UnixPath.SEPARATOR.toInt, path.length -2) else path.lastIndexOf(UnixPath.SEPARATOR.toInt) - index match { - case -1 => if (isAbsolute) Some(UnixPath.ROOT_PATH) else None - case pos => Some(UnixPath(path.substring(0, pos + 1))) - } - } - - def getRoot: Option[UnixPath] = if (isAbsolute) Some(UnixPath.ROOT_PATH) else None - - def subPath(beginIndex: Int, endIndex: Int): Try[UnixPath] = { - if (path.isEmpty() && beginIndex == 0 && endIndex == 1) { - return Success(this) - } - - if (beginIndex < 0 || endIndex < beginIndex) { - return Failure(new IllegalArgumentException(s"begin index or end index is invalid")) - } - - Try(UnixPath(parts.slice(beginIndex, endIndex).mkString(UnixPath.SEPARATOR.toString))) - } - - def getNameCount: Int = { - if (path.isEmpty()) { - 1 - } else if (isRoot) { - 0 - } else { - parts.size - } - } - - def getName(index: Int): Try[UnixPath] = { - if (path.isEmpty()){ - return Failure(new IllegalArgumentException("can not get name from a empty path")) - } - - if (index > length - 1) { - return Failure(new IndexOutOfBoundsException(s"index ${index} out of name count ${length -1}")) - } - - return Success(UnixPath(parts(2))) - } - - def resolve(other: UnixPath): UnixPath = { - if (other.path.isEmpty()){ - this - } else if (other.isAbsolute) { - other - } else if (hasTrailingSeparator) { - new UnixPath(path + other.path) - } else { - new UnixPath(path + UnixPath.SEPARATOR.toString + other.path) - } - } - - def resolveSibling(other: UnixPath): UnixPath = { - getParent match { - case Some(parent: UnixPath) => - parent.resolve(other) - case None => other - } - } - - def relativize(other: UnixPath): UnixPath = { - if (path.isEmpty()){ - return other - } - - val left = split().buffered - val right = other.split().buffered - breakable( - while (left.hasNext && right.hasNext){ - if (!(left.head == right.head)){ - break() - } - - left.next() - right.next() - } - ) - - val result = new StringBuilder(path.length + other.path.length) - while (left.hasNext){ - result.append(UnixPath.PARENT_DIR) - result.append(UnixPath.SEPARATOR) - left.next() - } - - while (right.hasNext) { - result.append(right.next()) - result.append(UnixPath.SEPARATOR) - } - - if (result.length > 0 && !other.hasTrailingSeparator) { - result.deleteCharAt(result.length - 1) - } - - return new UnixPath(result.toString) - } - - def normalize(): UnixPath = { - val parts = mutable.ArrayBuffer[String]() - var mutated = false - var resultLength = 0 - var mark = 0 - var index = 0 - val current = UnixPath.CURRENT_DIR + UnixPath.SEPARATOR.toString - val parent = UnixPath.PARENT_DIR + UnixPath.SEPARATOR - do { - index = path.indexOf(UnixPath.SEPARATOR.toInt, mark) - val part = path.substring(mark, if (index == -1) path.length else index + 1) - part match { - case UnixPath.CURRENT_DIR | `current` => mutated = true - case UnixPath.PARENT_DIR | `parent` => - mutated = true - if (!parts.isEmpty){ - resultLength -= parts.remove(parts.length -1).length - } - case _ => - if (index != mark || index == 0) { - parts.append(part) - resultLength += part.length - } else { - mutated = true - } - } - mark = index + 1 - } while (index != -1) - - if (!mutated){ - return this - } - - val result = new StringBuilder(resultLength) - - parts.foreach {part => result.append(part)} - - return new UnixPath(result.toString) - } - - def split(): Iterator[String] = parts.iterator - - def splitReverse(): Iterator[String] = parts.reverseIterator - - def removeBeginningSeparator(): UnixPath = { - if (isAbsolute) new UnixPath(path.substring(1)) else this - } - - def addTrailingSeparator(): UnixPath = { - if (hasTrailingSeparator) this else new UnixPath(path + UnixPath.SEPARATOR) - } - - def removeTrailingSeparator()(): UnixPath = { - if (!isRoot && hasTrailingSeparator) { - new UnixPath(path.substring(0, length -1)) - } else { - this - } - } - - def startsWith(other: UnixPath): Boolean = { - val me = removeTrailingSeparator()() - val oth = other.removeTrailingSeparator()() - - if (oth.path.length > me.path.length) { - return false - } else if (me.isAbsolute != oth.isAbsolute) { - return false - } else if (!me.path.isEmpty() && oth.path.isEmpty()) { - return false - } - - return startsWith(split(), other.split()) - } - - def startsWith(left: Iterator[String], right: Iterator[String]): Boolean = { - while (right.hasNext){ - if (!left.hasNext || right.next() != left.next()) { - return false - } - } - return true - } - - def endsWith(other: UnixPath): Boolean = { - val me = removeTrailingSeparator()() - val oth = other.removeTrailingSeparator()() - - if (oth.path.length > me.path.length) { - return false - } else if (!me.path.isEmpty() && oth.path.isEmpty()) { - return false - } else if (oth.isAbsolute) { - return me.isAbsolute && me.path == other.path - } - - startsWith(me.splitReverse(), other.splitReverse()) - } - - def toAbsolutePath(currentWorkingDirectory: UnixPath): Try[UnixPath] = { - if (!currentWorkingDirectory.isAbsolute) { - return Failure(new IllegalArgumentException(s"Not an absolute path ${currentWorkingDirectory}")) - } - - if (isAbsolute) Success(this) else Success(currentWorkingDirectory.resolve(this)) - } - - def toAbsolutePath(): UnixPath = { - if (isAbsolute) this else UnixPath.ROOT_PATH.resolve(this) - } - - def compareTo(other: UnixPath): Int = { - val me = parts.toList - val that = other.parts.toList - - if (me == that) { - return 0 - } else if (me < that) { - return -1 - } else { - return 1 - } - } - - override def equals(obj: scala.Any): Boolean = { - (this eq obj.asInstanceOf[AnyRef]) || { obj.isInstanceOf[UnixPath] && obj.asInstanceOf[UnixPath].path.equals(path)} - } - - override def length(): Int = { - path.length - } - - override def charAt(index: Int): Char = { - path.charAt(index) - } - - override def subSequence(start: Int, end: Int): CharSequence = { - path.subSequence(start, end) - } - - override def toString: String = { - path - } - - def initParts(): Array[String] = { - if (path.isEmpty()) { - Array.empty[String] - } else { - if (path.charAt(0) == UnixPath.SEPARATOR){ - path.substring(1).split(UnixPath.SEPARATOR) - } else { - path.split(UnixPath.SEPARATOR) - } - } - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/OssPathBuilderSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/OssPathBuilderSpec.scala deleted file mode 100644 index 10e664504f1..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/OssPathBuilderSpec.scala +++ /dev/null @@ -1,82 +0,0 @@ -package cromwell.filesystems.oss - -import com.typesafe.config.ConfigFactory -import cromwell.core.TestKitSuite -import cromwell.filesystems.oss.nio.OssNioUtilSpec -import org.scalatest.BeforeAndAfter -import org.scalatest.TryValues._ -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers - -object OssPathBuilderSpec { - - val BcsBackendConfigWithRefreshString = - s""" - | refresh-interval = 1800 - | auth { - | endpoint = "oss-cn-shanghai.aliyuncs.com" - | access-id = "test-access-id" - | access-key = "test-access-key" - | security-token = "test-security-token" - | } - | caching { - | duplication-strategy = "reference" - | } - """.stripMargin - - val BcsBackendConfigWithRefresh = ConfigFactory.parseString(BcsBackendConfigWithRefreshString) - - val BcsBackendConfigWithoutRefreshString = - s""" - | auth { - | endpoint = "oss-cn-shanghai.aliyuncs.com" - | access-id = "test-access-id" - | access-key = "test-access-key" - | } - | caching { - | duplication-strategy = "reference" - | } - """.stripMargin - - val BcsBackendConfigWithoutRefresh = ConfigFactory.parseString(BcsBackendConfigWithoutRefreshString) -} - -class OssPathBuilderSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with OssNioUtilSpec with BeforeAndAfter { - - behavior of "OssPathBuilerSpec" - val testPathBuiler = OssPathBuilder(mockOssConf) - - it should "throw when no bucket in URI" in { - testPathBuiler.build("oss:").failed.get shouldBe an[IllegalArgumentException] - testPathBuiler.build("oss://").failed.get shouldBe an[IllegalArgumentException] - } - - it should "throw when path has an invalid schema" in { - testPathBuiler.build(s"gcs://$bucket$fileName").failed.get shouldBe an[IllegalArgumentException] - } - - it should "has an empty key when no path specified" in { - testPathBuiler.build(s"oss://$bucket").success.value.bucket shouldBe bucket - testPathBuiler.build(s"oss://$bucket").success.value.key shouldBe empty - } - - it should "start with separator when path specified" in { - val path = testPathBuiler.build(s"oss://$bucket$fileName").success.value - path.bucket shouldBe bucket - path.nioPath.toString shouldBe fileName - path.key shouldBe fileName.stripPrefix("/") - path.pathAsString shouldBe s"oss://$bucket$fileName" - path.pathWithoutScheme shouldBe s"$bucket$fileName" - } - - it should "success from config" in { - val ossPathBuilder = OssPathBuilder.fromConfig(OssPathBuilderSpec.BcsBackendConfigWithRefresh, null) - ossPathBuilder.build(s"oss://$bucket").success.value.bucket shouldBe bucket - ossPathBuilder.build(s"oss://$bucket").success.value.key shouldBe empty - - val ossPathBuilderWithoutRefresh = OssPathBuilder.fromConfig(OssPathBuilderSpec.BcsBackendConfigWithoutRefresh, null) - ossPathBuilderWithoutRefresh.build(s"oss://$bucket").success.value.bucket shouldBe bucket - ossPathBuilderWithoutRefresh.build(s"oss://$bucket").success.value.key shouldBe empty - } - - } diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssAppendOutputStreamSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssAppendOutputStreamSpec.scala deleted file mode 100644 index ea72793dcdb..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssAppendOutputStreamSpec.scala +++ /dev/null @@ -1,42 +0,0 @@ -package cromwell.filesystems.oss.nio - -import cromwell.core.TestKitSuite - -class OssAppendOutputStreamSpec extends TestKitSuite with OssNioUtilSpec { - - behavior of s"OssAppendOutputStream" - - "write batch" should "work" taggedAs NeedAK in { - val path = OssStoragePath.getPath(ossFileSystem, "/test-oss-append") - val stream = OssAppendOutputStream(ossClient, path, true) - - val content: String = "haha" - stream.write(content.getBytes) - - contentAsString(path) shouldEqual content - stream.position shouldEqual content.length - } - - "write single" should "work" taggedAs NeedAK in { - val c: Char = 'c' - val path = OssStoragePath.getPath(ossFileSystem, "/test-oss-append") - val stream = OssAppendOutputStream(ossClient, path, true) - - stream.write(c.toInt) - - contentAsString(path) shouldEqual c.toString - stream.position shouldEqual 1 - } - - "write range" should "work" taggedAs NeedAK in { - val path = OssStoragePath.getPath(ossFileSystem, "/test-oss-append") - val stream = OssAppendOutputStream(ossClient, path, true) - - val content: String = "haha" - stream.write(content.getBytes, 1, 1) - - contentAsString(path) shouldEqual 'a'.toString - stream.position shouldEqual 1 - } - -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssFileReadChannelSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssFileReadChannelSpec.scala deleted file mode 100644 index 1c57460ef51..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssFileReadChannelSpec.scala +++ /dev/null @@ -1,80 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.charset.Charset - -import cromwell.core.TestKitSuite -import org.scalatest.{BeforeAndAfter} - -import scala.util.Try -import scala.util.control.Breaks - -object OssFileReadChannelSpec { - val FILENAME = "/test-oss-read-file" - val CONTENT = "Hello World!" - - implicit class Crossable[X](xs: Iterable[X]) { - def cross[Y](ys: Iterable[Y]) = for { x <- xs; y <- ys } yield (x, y) - } -} - -class OssFileReadChannelSpec extends TestKitSuite with OssNioUtilSpec with BeforeAndAfter { - behavior of s"OssFileReadChannelSpec" - - import OssFileReadChannelSpec._ - - - def getPath = OssStoragePath.getPath(ossFileSystem, FILENAME) - - before { - Try(OssAppendOutputStream(ossClient, getPath, true)) foreach {_.write(CONTENT.getBytes("UTF-8"))} - } - - after { - Try(deleteObject(getPath)) - } - - it should "has the right size" taggedAs NeedAK in { - val channel = OssFileReadChannel(ossClient, 0L, getPath) - channel.size shouldEqual(CONTENT.length) - } - - it should "has the right content" taggedAs NeedAK in { - List.range(1, CONTENT.length + 1) foreach { bufferSize =>verifySameContent(bufferSize)} - for (bufferSize <- List.range(1, CONTENT.length + 1); position <- List.range(0, CONTENT.length)) { - verifySameContent(bufferSize, position.toLong) - } - } - - it should "has the right position after seeking" taggedAs NeedAK in { - val channel = OssFileReadChannel(ossClient, 0L, getPath) - channel.size shouldEqual(CONTENT.length) - - channel.position(1) - - channel.position shouldEqual(1) - } - - def verifySameContent(bufferSize: Int, position: Long = 0) = { - val channel = OssFileReadChannel(ossClient, position, getPath) - - import java.nio.ByteBuffer - val buf = ByteBuffer.allocate(bufferSize) - - val loop = new Breaks - val builder = new StringBuilder - - var bytesRead = channel.read(buf) - loop.breakable { - while (bytesRead != -1) { - buf.flip() - val charset = Charset.forName("UTF-8"); - - builder.append(charset.decode(buf).toString()) - buf.clear - bytesRead = channel.read(buf) - } - } - - builder.toString shouldEqual CONTENT.substring(position.toInt) - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssNioUtilSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssNioUtilSpec.scala deleted file mode 100644 index b4ebb84b7fa..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssNioUtilSpec.scala +++ /dev/null @@ -1,110 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.io.ByteArrayInputStream - -import com.aliyun.oss.OSSClient -import common.assertion.CromwellTimeoutSpec -import org.scalatest._ -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers - -import scala.util.control.Breaks -import scala.util.{Failure, Success, Try} - -object NeedAK extends Tag("this test need oss storage access id and key") - -object OssNioUtilSpec { - val DEFAULT_BUCKET = "bcs-bucket" - - val DEFAULT_FILE_NAME = "/bcs-dir/bcs-file" - - val DEFAULT_CONTENT = "Hello World!" - - val ossInfo: Map[String, String] = Map( - "endpoint" -> "", - "access-id" -> "", - "access-key" -> "", - "bucket" -> DEFAULT_BUCKET - ) -} - -trait OssNioUtilSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matchers { - - override def withFixture(test: NoArgTest): Outcome = { - if (test.tags.contains(NeedAK.name)) { - Try(ossConf) match { - case Success(_) => super.withFixture(test) - case Failure(_) => cancel(NeedAK.name) - } - } else { - super.withFixture(test) - } - } - - import OssNioUtilSpec._ - - lazy val bucket: String = { - val bucket = ossInfo.getOrElse("bucket", "mock-bucket") - if (bucket.isEmpty) { - throw new IllegalArgumentException("test bucket can not be empty") - } - - bucket - } - - lazy val ossConf: OssStorageConfiguration = Try{ - OssStorageConfiguration.parseMap(ossInfo) - } getOrElse(throw new IllegalArgumentException("you should supply oss info before testing oss related operation")) - - lazy val mockOssConf: OssStorageConfiguration = - DefaultOssStorageConfiguration("mock-endpoint", "mock-id", "mock-key", None) - - lazy val ossProvider: OssStorageFileSystemProvider = OssStorageFileSystemProvider(ossConf) - lazy val mockProvider: OssStorageFileSystemProvider = OssStorageFileSystemProvider(mockOssConf) - lazy val ossFileSystem: OssStorageFileSystem = OssStorageFileSystem(bucket, ossConf) - lazy val mockFileSystem: OssStorageFileSystem = OssStorageFileSystem(bucket, mockOssConf) - val fileName: String = DEFAULT_FILE_NAME - val fileContent: String = DEFAULT_CONTENT - - lazy val ossClient: OSSClient = mockOssConf.newOssClient() - - def contentAsString(path: OssStoragePath): String = { - val ossObject = ossClient.getObject(path.bucket, path.key) - - val in = OssStorageRetry.from( - () => ossObject.getObjectContent - ) - - val maxLen = 1024 - val loop = new Breaks - val result = new StringBuilder - loop.breakable { - while(true) { - val b = new Array[Byte](maxLen) - val got = OssStorageRetry.from( - () => in.read(b, 0, maxLen) - ) - if (got <= 0) { - loop.break() - } - result.append(new String(b, 0, got)) - } - } - - result.toString() - } - - def deleteObject(path: OssStoragePath): Unit = { - OssStorageRetry.from( - () => ossClient.deleteObject(path.bucket, path.key) - ) - () - } - - def writeObject(path: OssStoragePath): Unit = { - OssStorageRetry.from{ - () => ossClient.putObject(path.bucket, path.key, new ByteArrayInputStream(fileContent.getBytes())) - } - () - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributesViewSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributesViewSpec.scala deleted file mode 100644 index 7825a369076..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileAttributesViewSpec.scala +++ /dev/null @@ -1,39 +0,0 @@ -package cromwell.filesystems.oss.nio - -import com.aliyun.oss.OSSClient -import com.aliyun.oss.model.GenericRequest -import org.mockito.Mockito._ -import org.mockito.ArgumentMatchers._ - - -class OssStorageFileAttributesViewSpec extends OssNioUtilSpec { - behavior of "OssStorageFileAttributesView" - - import OssStorageObjectAttributesSpec._ - - private def getObject = { - OssStoragePath.getPath(mockFileSystem, fileName) - } - - private def getDir = { - OssStoragePath.getPath(mockFileSystem, "/bcs-dir/") - } - - it should "return an object attr" in { - val ossClient = mock[OSSClient] - when(ossClient.doesObjectExist(any[GenericRequest]())).thenReturn(true) - val meta = getObjectMeta - when(ossClient.getObjectMetadata(anyString(), anyString())).thenReturn(meta) - - val view = OssStorageFileAttributesView(ossClient, getObject) - view.readAttributes shouldBe an [OssStorageObjectAttributes] - } - - it should "return an dir attr" in { - val ossClient = mock[OSSClient] - when(ossClient.doesObjectExist(any[GenericRequest]())).thenReturn(true) - val view = OssStorageFileAttributesView(ossClient, getDir) - view.readAttributes shouldBe a [OssStorageDirectoryAttributes] - } - -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemProviderSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemProviderSpec.scala deleted file mode 100644 index 59638e36878..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemProviderSpec.scala +++ /dev/null @@ -1,194 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.net.URI -import java.nio.charset.Charset -import java.nio.file.{DirectoryStream, NoSuchFileException, Path, StandardOpenOption} - -import cromwell.core.TestKitSuite -import org.scalatest.BeforeAndAfter - -import scala.jdk.CollectionConverters._ -import scala.collection.mutable.ArrayBuffer -import scala.util.control.Breaks - -class OssStorageFileSystemProviderSpec extends TestKitSuite with OssNioUtilSpec with BeforeAndAfter { - behavior of "OssStorageFileSystemProviderSpec" - - it should "has right schema" in { - mockProvider.getScheme shouldEqual OssStorageFileSystem.URI_SCHEMA - } - - it should "work when creating new file system" in { - val fs = mockProvider.newFileSystem(URI.create(s"oss://$bucket"), mockOssConf.toMap.asJava) - fs.bucket shouldEqual bucket - - an [IllegalArgumentException] should be thrownBy ossProvider.newFileSystem(URI.create(s"oss://"), mockOssConf.toMap.asJava) - an [IllegalArgumentException] should be thrownBy ossProvider.newFileSystem(URI.create(s"oss://$bucket:8812"), mockOssConf.toMap.asJava) - - val fs1 = mockProvider.getFileSystem(URI.create(s"oss://$bucket")) - fs1.bucket shouldEqual bucket - } - - it should "work when getting a new oss path" in { - val path = mockProvider.getPath(URI.create(s"oss://$bucket$fileName")) - path.bucket shouldEqual bucket - path.key shouldEqual fileName.stripPrefix(OssStorageFileSystem.SEPARATOR) - } - - it should "work when creating an output stream" taggedAs NeedAK in { - val path = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - - val outS = ossProvider.newOutputStream(path) - outS.write(fileContent.getBytes) - - contentAsString(path) shouldEqual fileContent - outS.asInstanceOf[OssAppendOutputStream].position shouldEqual fileContent.length - } - - it should "work when creating an byte channel" taggedAs NeedAK in { - val path = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - - val outS = ossProvider.newOutputStream(path) - outS.write(fileContent.getBytes) - - val inC = ossProvider.newByteChannel(path, Set(StandardOpenOption.READ).asJava) - - import java.nio.ByteBuffer - val buf = ByteBuffer.allocate(1) - - val loop = new Breaks - val builder = new StringBuilder - - var bytesRead = inC.read(buf) - loop.breakable { - while (bytesRead != -1) { - buf.flip() - val charset = Charset.forName("UTF-8") - - builder.append(charset.decode(buf).toString) - buf.clear - bytesRead = inC.read(buf) - } - } - - builder.toString shouldEqual fileContent - } - - it should "delete file if it exists" taggedAs NeedAK in { - val path = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - - val outS = ossProvider.newOutputStream(path) - outS.write(fileContent.getBytes) - outS.close() - - ossProvider.deleteIfExists(path) shouldEqual true - ossProvider.deleteIfExists(path) shouldEqual false - an [NoSuchFileException] should be thrownBy ossProvider.delete(path) - } - - it should "work when copying an object" taggedAs NeedAK in { - val src = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - val target = ossProvider.getPath(URI.create(s"oss://$bucket${fileName}1")) - ossProvider.deleteIfExists(src) - - writeObject(src) - - ossProvider.copy(src, target) - - ossProvider.deleteIfExists(target) shouldEqual true - } - - it should "work when moving an object" taggedAs NeedAK in { - val src = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - val target = ossProvider.getPath(URI.create(s"oss://$bucket${fileName}1")) - ossProvider.deleteIfExists(src) - - writeObject(src) - - ossProvider.move(src, target) - - ossProvider.deleteIfExists(target) shouldEqual true - ossProvider.deleteIfExists(src) shouldEqual false - - } - - it should "work for some basic operations" taggedAs NeedAK in { - val path = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - val path1 = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - - ossProvider.isHidden(path) shouldEqual false - ossProvider.isSameFile(path, path1) - - an [UnsupportedOperationException] should be thrownBy ossProvider.getFileStore(path) - - an [NoSuchFileException] should be thrownBy ossProvider.checkAccess(path) - - val dir = ossProvider.getPath(URI.create(s"oss://$bucket${fileName}/")) - noException should be thrownBy ossProvider.checkAccess(dir) - } - - it should "work for attribute view" taggedAs NeedAK in { - val path = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - ossProvider.deleteIfExists(path) - - writeObject(path) - val view = ossProvider.getFileAttributeView(path, classOf[OssStorageFileAttributesView]) - view shouldBe an [OssStorageFileAttributesView] - - val attr = view.readAttributes() - attr shouldBe an [OssStorageObjectAttributes] - - val dir = ossProvider.getPath(URI.create(s"oss://$bucket${fileName}/")) - val dirView = ossProvider.getFileAttributeView(dir, classOf[OssStorageFileAttributesView]) - dirView shouldBe an [OssStorageFileAttributesView] - - val dirAttr = dirView.readAttributes() - dirAttr shouldBe an [OssStorageDirectoryAttributes] - } - - it should "work for reading attrs" taggedAs NeedAK in { - val path = ossProvider.getPath(URI.create(s"oss://$bucket$fileName")) - ossProvider.deleteIfExists(path) - - writeObject(path) - val attr = ossProvider.readAttributes(path, classOf[OssStorageFileAttributes]) - attr shouldBe an [OssStorageObjectAttributes] - - ossProvider.deleteIfExists(path) - a [NoSuchFileException] should be thrownBy ossProvider.readAttributes(path, classOf[OssStorageFileAttributes]) - - val dir = ossProvider.getPath(URI.create(s"oss://$bucket${fileName}/")) - val dirAttr = ossProvider.readAttributes(dir, classOf[OssStorageFileAttributes]) - dirAttr shouldBe an [OssStorageDirectoryAttributes] - } - - - it should "work for reading dirs" taggedAs NeedAK in { - val count = 10 - val testDir = "/test-read-dir" - val filePrefix = "test-file" - val expectedFileNames = ArrayBuffer.empty[String] - val dir = ossProvider.getPath(URI.create(s"oss://$bucket$testDir/")) - for (i <- 0 to count) { - val fileName = filePrefix + i.toString - expectedFileNames.append(fileName) - - val path = dir.resolve(fileName) - - ossProvider.deleteIfExists(path) - writeObject(path) - } - - val dirStream = ossProvider.newDirectoryStream(dir, new DirectoryStream.Filter[Path] { - override def accept(entry: Path): Boolean = { - true - } - }) - - val files = ArrayBuffer.empty[String] - dirStream.iterator.asScala foreach(file => files.append(file.toString)) - - files should contain allElementsOf(expectedFileNames) - } - -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemSpec.scala deleted file mode 100644 index ea05cfa8bf9..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageFileSystemSpec.scala +++ /dev/null @@ -1,30 +0,0 @@ -package cromwell.filesystems.oss.nio - -import cromwell.core.TestKitSuite - -class OssStorageFileSystemSpec extends TestKitSuite with OssNioUtilSpec { - behavior of s"OssStorageFileSystemSpec" - - it should "get right path" in { - val ossPath = mockFileSystem.getPath("/test-file-system") - ossPath.bucket shouldEqual(bucket) - ossPath.key shouldEqual("test-file-system") - } - - it should "has right view name" in { - val fs = mockFileSystem - - fs.supportedFileAttributeViews should contain (OssStorageFileSystem.BASIC_VIEW) - fs.supportedFileAttributeViews should contain (OssStorageFileSystem.OSS_VIEW) - } - - it should "do not support some method" in { - an [UnsupportedOperationException] should be thrownBy mockFileSystem.newWatchService - } - - it should "return some expected simple mocked result" in { - mockFileSystem.isOpen shouldBe true - mockFileSystem.isReadOnly shouldBe false - mockFileSystem.getSeparator shouldBe OssStorageFileSystem.SEPARATOR - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageObjectAttributesSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageObjectAttributesSpec.scala deleted file mode 100644 index 4b2ab275b4d..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageObjectAttributesSpec.scala +++ /dev/null @@ -1,92 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.nio.file.attribute.FileTime -import com.aliyun.oss.model.ObjectMetadata -import common.mock.MockSugar -import cromwell.core.TestKitSuite -import org.mockito.Mockito._ - -import java.text.SimpleDateFormat -import java.util.{Date, Locale} - -object OssStorageObjectAttributesSpec extends MockSugar { - val DEFAULT_BUCKET = "bcs-bucket" - - val DEFAULT_FILE_NAME = "/bcs-dir/bcs-file" - - val DEFAULT_LENGTH: Long = 2102784 - - val DEFAULT_MODIFIED: Date = { - val target = "Thu Dec 21 15:19:27 CST 2017" - val df = new SimpleDateFormat("EEE MMM dd kk:mm:ss z yyyy", Locale.ENGLISH) - df.parse(target) - } - - val DEFAULT_ETAG = "F80066F040BDA4F991DB5F8AEC9905FB" - - val DEFAULT_CONTENT_DISPOSITION: String = null - - val DEFAULT_CACHE_CONTROL: String = null - - val DEFAULT_CONTENT_ENCODING: String = null - - val DEFAULT_CONTENT_TYPE = "application/x-msdownload" - - def getObjectMeta: ObjectMetadata = { - val meta = mock[ObjectMetadata] - - when(meta.getContentDisposition).thenReturn(DEFAULT_CONTENT_DISPOSITION) - when(meta.getContentEncoding).thenReturn(DEFAULT_CONTENT_ENCODING) - when(meta.getCacheControl).thenReturn(DEFAULT_CACHE_CONTROL) - when(meta.getLastModified).thenReturn(DEFAULT_MODIFIED) - when(meta.getETag).thenReturn(DEFAULT_ETAG) - when(meta.getContentType).thenReturn(DEFAULT_CONTENT_TYPE) - when(meta.getContentLength).thenReturn(DEFAULT_LENGTH) - when(meta.getExpirationTime).thenThrow(new NullPointerException()) - - meta - } -} - -class OssStorageObjectAttributesSpec extends TestKitSuite with OssNioUtilSpec { - - behavior of s"OssStorageObjectAttributes" - - import OssStorageObjectAttributesSpec._ - - def getObject: OssStoragePathImpl = { - OssStoragePath.getPath(mockFileSystem, fileName) - } - - def getDir: OssStoragePathImpl = { - OssStoragePath.getPath(mockFileSystem, "/bcs-dir/") - } - - "an oss object attr" should "be an right" in { - val attr = OssStorageObjectAttributes(getObjectMeta, getObject) - - attr.fileKey shouldEqual getObject.pathAsString - - attr.creationTime shouldEqual attr.lastModifiedTime() - attr.lastAccessTime shouldEqual FileTime.fromMillis(0) - attr.cacheControl() shouldBe empty - attr.contentDisposition shouldBe empty - attr.contentEncoding shouldBe empty - attr.etag shouldBe Option(DEFAULT_ETAG) - attr.size shouldBe DEFAULT_LENGTH - } - - "an oss directory attr" should "be an right" in { - val attr = OssStorageDirectoryAttributes(getDir) - - attr.fileKey shouldEqual getDir.pathAsString - - attr.creationTime shouldEqual attr.lastModifiedTime() - attr.lastAccessTime shouldEqual FileTime.fromMillis(0) - attr.cacheControl() shouldBe empty - attr.contentDisposition shouldBe empty - attr.contentEncoding shouldBe empty - attr.etag shouldBe empty - attr.size shouldBe 0 - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStoragePathSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStoragePathSpec.scala deleted file mode 100644 index 74008bdeb27..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStoragePathSpec.scala +++ /dev/null @@ -1,65 +0,0 @@ -package cromwell.filesystems.oss.nio - -import cromwell.core.TestKitSuite -import scala.jdk.CollectionConverters._ - - -class OssStoragePathSpec extends TestKitSuite with OssNioUtilSpec { - behavior of s"OssStoragePath" - - - it should s"has the same bucket with file system" in { - - val path = OssStoragePath.getPath(mockFileSystem, fileName) - - path.bucket shouldBe bucket - - path.toAbsolutePath.toString shouldBe fileName - } - - it should s"has a separator-removed key" in { - val path = OssStoragePath.getPath(mockFileSystem, fileName) - - path.key shouldBe fileName.stripPrefix(UnixPath.SEPARATOR.toString) - } - - "a not absolute oss path" should s"has a NullOssStoragePath root path" in { - val path = OssStoragePath.getPath(mockFileSystem, fileName.stripPrefix(UnixPath.SEPARATOR.toString)) - - path.getRoot shouldBe a [NullOssStoragePath] - } - - "an absolute oss path" should s"has a OssStoragePathImpl root path" in { - val path = OssStoragePath.getPath(mockFileSystem, fileName) - - path.getRoot shouldBe an [OssStoragePathImpl] - } - - it should s"has right iterator" in { - val path = OssStoragePath.getPath(mockFileSystem, fileName) - - var subs = List.empty[String] - path.iterator().asScala foreach(p => subs = subs :+ p.toString) - - subs.head shouldBe "bcs-dir" - subs(1) shouldBe "bcs-file" - } - - it should s"has right relativize" in { - val path = OssStoragePath.getPath(mockFileSystem, fileName) - - val path1 = OssStoragePath.getPath(mockFileSystem, "/bcs-dir/bcs-file1") - - path.relativize(path1).toString shouldEqual "../bcs-file1" - - val path2 = OssStoragePath.getPath(mockFileSystem, "/bcs-dir1/bcs-file2") - path.relativize(path2).toString shouldEqual "../../bcs-dir1/bcs-file2" - } - - it should s"has right pathAsString" in { - val path = OssStoragePath.getPath(mockFileSystem, fileName) - - path.pathAsString shouldEqual s"oss://$bucket$fileName" - } - -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageRetrySpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageRetrySpec.scala deleted file mode 100644 index ea2f03e0605..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/OssStorageRetrySpec.scala +++ /dev/null @@ -1,126 +0,0 @@ -package cromwell.filesystems.oss.nio - -import cromwell.core.TestKitSuite - -import scala.util.{Success, Failure, Try} - -case class FatalError(message: String = "", cause: Throwable = None.orNull) extends Exception(message, cause) -case class TransientError(message: String = "", cause: Throwable = None.orNull) extends Exception(message, cause) -case class RetryableError(message: String = "", cause: Throwable = None.orNull) extends Exception(message, cause) - -class RetryContext { - var retried = 0 - - def doSth(f: Int => Try[Int]): Unit = { - f(retried) match { - case Success(_) => retried += 1 - case Failure(e: RetryableError) => - retried += 1 - throw e - case Failure(e: TransientError) => - retried += 1 - throw e - case Failure(e) => throw e - } - } -} - -object OssStorageRetrySpec { - def isFatal(t: Throwable): Boolean = t match { - case _: FatalError => true - case _ => false - } - - def isTransient(t: Throwable): Boolean = t match { - case _: TransientError => true - case _ => false - } -} - -class OssStorageRetrySpec extends TestKitSuite with OssNioUtilSpec { - - import OssStorageRetrySpec._ - - behavior of s"OssStorageRetrySpec" - - it should "retry throw immediately when fatal error occours" in { - val f = (x: Int) => if (x == 0) Failure(new FatalError) else Success(x) - val ctx = new RetryContext() - an [FatalError] should be thrownBy OssStorageRetry.fromTry( - () => Try{ - ctx.doSth(f) - }, - isFatal = isFatal, - isTransient = isTransient - ) - - ctx.retried shouldEqual(0) - } - - it should "retry if non-fatal error occurs" in { - val needRetry = 5 - val f = (x: Int) => { - if (x < needRetry) { - Failure(new RetryableError()) - } else { - Failure(new FatalError()) - } - } - - val ctx = new RetryContext() - an [FatalError] should be thrownBy OssStorageRetry.fromTry( - () => Try{ - ctx.doSth(f) - }, - isFatal = isFatal, - isTransient = isTransient - ) - - ctx.retried shouldEqual(needRetry) - } - - it should "success after retry max retries " in { - - val needRetry = 5 - val f = (x: Int) => { - if (x < needRetry) { - Failure(new RetryableError()) - } else { - Success(x) - } - } - - val ctx = new RetryContext() - OssStorageRetry.fromTry( - () => Try{ - ctx.doSth(f) - }, - isFatal = isFatal, - isTransient = isTransient - ) - - ctx.retried shouldEqual(needRetry + 1) - } - - it should "retry at most max retry times" in { - val needRetry = OssStorageRetry.DEFAULT_MAX_RETRIES + 1 - val f = (x: Int) => { - if (x < needRetry) { - Failure(new RetryableError()) - } else { - Success(x) - } - } - - val ctx = new RetryContext() - an [RetryableError] should be thrownBy OssStorageRetry.fromTry( - () => Try{ - ctx.doSth(f) - }, - isFatal = isFatal, - isTransient = isTransient - ) - - ctx.retried shouldEqual(OssStorageRetry.DEFAULT_MAX_RETRIES) - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/TTLOssStorageConfigurationSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/TTLOssStorageConfigurationSpec.scala deleted file mode 100644 index 739760fea42..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/TTLOssStorageConfigurationSpec.scala +++ /dev/null @@ -1,50 +0,0 @@ -package cromwell.filesystems.oss.nio - -import java.net.URI -import com.typesafe.config.{Config, ConfigFactory} -import cromwell.core.TestKitSuite -import org.scalatest.BeforeAndAfter -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers - -object TTLOssStorageConfigurationSpec { - - val BcsBackendConfigString: String = - s""" - | auth { - | endpoint = "oss-cn-shanghai.aliyuncs.com" - | access-id = "test-access-id" - | access-key = "test-access-key" - | security-token = "test-security-token" - | } - | caching { - | duplication-strategy = "reference" - | } - """.stripMargin - - val BcsBackendConfig: Config = ConfigFactory.parseString(BcsBackendConfigString) -} - -class TTLOssStorageConfigurationSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with BeforeAndAfter { - val expectedEndpoint = "oss-cn-shanghai.aliyuncs.com" - val expectedAccessId = "test-access-id" - val expectedAccessKey = "test-access-key" - val expectedToken: Option[String] = Option("test-security-token") - val expectedFullEndpoint: URI = URI.create("http://oss-cn-shanghai.aliyuncs.com") - - behavior of "TTLOssStorageConfiguration" - - - it should "have correct OSS credential info" in { - - val ossConfig = TTLOssStorageConfiguration(TTLOssStorageConfigurationSpec.BcsBackendConfig) - - ossConfig.endpoint shouldEqual expectedEndpoint - ossConfig.accessId shouldEqual expectedAccessId - ossConfig.accessKey shouldEqual expectedAccessKey - ossConfig.securityToken shouldEqual expectedToken - - ossConfig.newOssClient().getEndpoint shouldEqual expectedFullEndpoint - - } -} diff --git a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/UnixPathSpec.scala b/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/UnixPathSpec.scala deleted file mode 100644 index de4ccf3bb17..00000000000 --- a/filesystems/oss/src/test/scala/cromwell/filesystems/oss/nio/UnixPathSpec.scala +++ /dev/null @@ -1,335 +0,0 @@ -package cromwell.filesystems.oss.nio - -import cromwell.core.TestKitSuite -import org.scalatest.TryValues._ - -import scala.util.{Failure, Success, Try} - -case class ValidPath(pathAsString: String, - parent: Option[String], - fileName: Option[String], - nameCount: Int, - root: Option[String] = Some(UnixPath.ROOT_PATH.toString), - isRoot: Boolean = false, - isAbsolute: Boolean = true, - hasTrailingSeparator: Boolean = false, - likeDir: Boolean = false, - ) - -case class SubPath(pathAsString: String, - beginIndex: Int, - endIndex: Int, - subPath: Try[String], - description: String = "" - ) - -case class ResolvePath(pathAsString: String, - other: String, - resolved: String, - description: String = "" - ) - -case class ResolveSiblingPath(pathAsString: String, - other: String, - resolved: String, - description: String = "" - ) - -case class NormalizePath(pathAsString: String, - normalized: String - ) - -case class RelativizePath(pathAsString: String, - other: String, - relative: String - ) - -class UnixPathSpec extends TestKitSuite with OssNioUtilSpec { - - validPaths foreach { path => - it should behave like verifyValidPath(path) - } - - subPaths foreach { path => - it should behave like verifySubPath(path) - } - - resolvePaths foreach { path => - it should behave like verifyResolvePath(path) - } - - resolveSiblingPaths foreach { path => - it should behave like verifyResolveSiblingPath(path) - } - - normalizePaths foreach { path => - it should behave like verifyNormalizePath(path) - } - - relativizePaths foreach { path => - it should behave like verifyRelativePath(path) - } - - - def verifyValidPath(path: ValidPath) = { - behavior of s"Verify a valid UnixPath ${path.pathAsString}" - - val clue = s"pathAsString: ${path.pathAsString}" - - val unixPath = UnixPath(path.pathAsString) - - it should "match expected parent" in - withClue(clue) { - unixPath.getParent map {_.toString} shouldEqual path.parent - } - - it should "match expected name count" in - withClue(clue) { - unixPath.getNameCount shouldEqual path.nameCount - } - - it should "match expected file name" in - withClue(clue) { - unixPath.getFileName map {_.toString} shouldEqual path.fileName - } - - it should "match expected root" in - withClue(clue) { - unixPath.getRoot map {_.toString} shouldEqual path.root - } - - it should "match expected isRoot" in - withClue(clue) { - unixPath.isRoot shouldBe path.isRoot - } - - it should "match expected isAbsolute" in - withClue(clue) { - unixPath.isAbsolute shouldBe path.isAbsolute - } - - it should "match expected hasTrailingSeparator" in - withClue(clue) { - unixPath.hasTrailingSeparator shouldBe path.hasTrailingSeparator - } - - it should "match expected seemsLikeDirectory" in - withClue(clue) { - unixPath.seemsLikeDirectory() shouldBe path.likeDir - } - } - - def verifySubPath(path: SubPath) = { - val clue = s"path ${path.pathAsString} beginIndex ${path.beginIndex} endIndex ${path.endIndex}" - - behavior of s"Verify a unix path's sub path ${clue}" - - val unixPath = UnixPath(path.pathAsString) - - it should "match sub path" in - withClue(clue) { - val maybeRes = unixPath.subPath(path.beginIndex, path.endIndex) - path.subPath match { - case Success(sub: String) => - maybeRes.success.value.toString shouldEqual(sub) - case Failure(_) => - maybeRes.failure - } - } - } - - def verifyResolvePath(path: ResolvePath) = { - val clue = s"path ${path.pathAsString} other ${path.other}" - - behavior of s"Verify resolving a path on a UnixPath ${clue}" - - val me = UnixPath(path.pathAsString) - val other = UnixPath(path.other) - - it should "match resolved path" in - withClue(clue) { - me.resolve(other).toString shouldEqual(path.resolved) - } - } - - def verifyResolveSiblingPath(path: ResolveSiblingPath) = { - val clue = s"path ${path.pathAsString} other ${path.other}" - - behavior of s"Verify resolving sibling on a UnixPath ${clue}" - - val me = UnixPath(path.pathAsString) - val other = UnixPath(path.other) - - it should "match expected sibling path" in - withClue(clue) { - me.resolveSibling(other).toString shouldEqual(path.resolved) - } - } - - - def verifyNormalizePath(path: NormalizePath) = { - val clue = s"path ${path.pathAsString}" - - behavior of s"Verify normalize a UnixPath ${clue}" - - val me = UnixPath(path.pathAsString) - - it should "match expected normalized path" in - withClue(clue) { - me.normalize().toString shouldEqual(path.normalized) - } - } - - - def verifyRelativePath(path: RelativizePath) = { - val clue = s"path ${path.pathAsString} other ${path.other}" - - behavior of s"Verify resolving relativize on a UnixPath ${clue}" - - val me = UnixPath(path.pathAsString) - val other = UnixPath(path.other) - - it should "match resolved path" in - withClue(clue) { - me.relativize(other).toString shouldEqual(path.relative) - } - } - - private[this] def validPaths = Seq( - ValidPath( - pathAsString = "/bcs-dir/bcs-file", - parent = Some("/bcs-dir/"), - fileName = Some("bcs-file"), - nameCount = 2 - ), - ValidPath( - pathAsString = "/bcs-dir/bcs-dir1/", - parent = Some("/bcs-dir/"), - fileName = Some("bcs-dir1"), - nameCount = 2, - hasTrailingSeparator = true, - likeDir = true - ), - ValidPath( - pathAsString = "bcs-file", - parent = None, - fileName = Some("bcs-file"), - nameCount = 1, - root = None, - isAbsolute = false, - ) - ) - - private[this] def subPaths = Seq( - SubPath( - pathAsString = "/bcs-dir/bcs-dir1/bcs-dir2", - beginIndex = 0, - endIndex = 1, - subPath = Success("bcs-dir"), - description = "valid slice" - ), - SubPath( - pathAsString = "/bcs-dir/bcs-dir1/bcs-dir2", - beginIndex = 1, - endIndex = 0, - subPath = Failure(new IllegalArgumentException()), - description = "invalid index" - ), - SubPath( - pathAsString = "/bcs-dir/bcs-dir1/bcs-dir2", - beginIndex = 1, - endIndex = 10, - subPath = Success("bcs-dir1/bcs-dir2"), - description = "valid index" - ), - SubPath( - pathAsString = "/bcs-dir/bcs-dir1/bcs-dir2", - beginIndex = 3, - endIndex = 10, - subPath = Success(""), - description = "valid index" - ) - ) - - private[this] def resolvePaths = Seq( - ResolvePath( - pathAsString = "/bcs-dir/bcs-dir1", - other = "", - resolved = "/bcs-dir/bcs-dir1" - ), - ResolvePath( - pathAsString = "/bcs-dir/bcs-dir1", - other = "/bcs-dir2/bcs-dir3", - resolved = "/bcs-dir2/bcs-dir3" - ), - ResolvePath( - pathAsString = "/bcs-dir/bcs-dir1/", - other = "bcs-file", - resolved = "/bcs-dir/bcs-dir1/bcs-file" - ), - ResolvePath( - pathAsString = "/bcs-dir/bcs-dir1", - other = "bcs-file", - resolved = "/bcs-dir/bcs-dir1/bcs-file" - ), - ) - - private[this] def resolveSiblingPaths = Seq( - ResolveSiblingPath( - pathAsString = "/bcs-dir/bcs-file1", - other = "bcs-file2", - resolved = "/bcs-dir/bcs-file2" - ), - ResolveSiblingPath( - pathAsString = "/", - other = "bcs-file2", - resolved = "bcs-file2" - ), - ResolveSiblingPath( - pathAsString = "", - other = "bcs-file2", - resolved = "bcs-file2" - ), - ResolveSiblingPath( - pathAsString = "/bcs-dir/bcs-file1", - other = "/bcs-file2", - resolved = "/bcs-file2" - ), - ) - - private[this] def normalizePaths = Seq( - NormalizePath( - "/bcs-dir/.", - "/bcs-dir/" - ), - NormalizePath( - "/bcs-dir/../bcs-file", - "/bcs-file" - ), - NormalizePath( - "/bcs-dir/./bcs-file", - "/bcs-dir/bcs-file" - ), - NormalizePath( - "/bcs-dir/./bcs-dir1/", - "/bcs-dir/bcs-dir1/" - ), - NormalizePath( - "../bcs-dir/bcs-dir1/", - "bcs-dir/bcs-dir1/" - ) - ) - - private[this] def relativizePaths = Seq( - RelativizePath( - "/bcs-dir1/bcs-file1", - "/bcs-dir1/bcs-file2", - "../bcs-file2" - ), - RelativizePath( - "/bcs-dir1/bcs-file1", - "/bcs-file2", - "../../bcs-file2" - ) - ) -} diff --git a/mkdocs.yml b/mkdocs.yml index fb06a7c544c..68e73560774 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,7 +12,6 @@ nav: - Quick Introduction: tutorials/FiveMinuteIntro.md - How to Configure Cromwell: tutorials/ConfigurationFiles.md - Getting started with Google Cloud: tutorials/PipelinesApi101.md - - Getting started with Alibaba Cloud: tutorials/BCSIntro.md - Getting started with AWS (beta): tutorials/AwsBatch101.md - View the Timing Diagrams: tutorials/TimingDiagrams.md - Persisting data between restarts: tutorials/PersistentServer.md @@ -56,7 +55,6 @@ nav: - Local: backends/Local.md - Google Cloud: backends/Google.md - AWS Batch: backends/AWSBatch.md - - Alibaba Cloud: backends/BCS.md - AWS Batch (beta): backends/AWS.md - GA4GH TES: backends/TES.md - HPC: backends/HPC.md diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2b3bc3fb574..0dbf9b4325f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -4,10 +4,6 @@ object Dependencies { private val akkaHttpCirceIntegrationV = "1.39.2" private val akkaHttpV = "10.1.15" // (CROM-6619) private val akkaV = "2.5.32" // scala-steward:off (CROM-6637) - private val aliyunBcsV = "6.2.4" - private val aliyunCoreV = "4.6.0" - private val aliyunCrV = "4.1.4" - private val aliyunOssV = "3.14.0" private val ammoniteOpsV = "2.4.1" private val apacheHttpClientV = "4.5.13" private val awsSdkV = "2.17.152" @@ -322,24 +318,6 @@ object Dependencies { exclude("com.google.guava", "guava-jdk5") ) ++ googleGenomicsV2Alpha1Dependency ++ googleLifeSciencesV2BetaDependency - private val aliyunOssDependencies = List( - "com.aliyun.oss" % "aliyun-sdk-oss" % aliyunOssV - exclude("com.sun.activation", "jakarta.activation") - ) - - private val aliyunBatchComputeDependencies = List( - "com.aliyun" % "aliyun-java-sdk-batchcompute" % aliyunBcsV, - "com.aliyun" % "aliyun-java-sdk-core" % aliyunCoreV - exclude("com.sun.activation", "jakarta.activation") - ) - - private val aliyunCrDependencies = List( - "com.aliyun" % "aliyun-java-sdk-cr" % aliyunCrV, - "com.aliyun" % "aliyun-java-sdk-core" % aliyunCoreV - exclude("com.sun.activation", "jakarta.activation"), - "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpV - ) - private val dbmsDependencies = List( "org.hsqldb" % "hsqldb" % hsqldbV, "org.mariadb.jdbc" % "mariadb-java-client" % mariadbV, @@ -419,10 +397,6 @@ object Dependencies { val httpFileSystemDependencies: List[ModuleID] = akkaHttpDependencies - val ossFileSystemDependencies: List[ModuleID] = googleCloudDependencies ++ aliyunOssDependencies ++ List( - "com.github.pathikrit" %% "better-files" % betterFilesV - ) - val womDependencies: List[ModuleID] = List( "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingV, "io.spray" %% "spray-json" % sprayJsonV, @@ -497,7 +471,7 @@ object Dependencies { val databaseMigrationDependencies: List[ModuleID] = liquibaseDependencies ++ dbmsDependencies - val dockerHashingDependencies: List[ModuleID] = http4sDependencies ++ circeDependencies ++ aliyunCrDependencies + val dockerHashingDependencies: List[ModuleID] = http4sDependencies ++ circeDependencies val cromwellApiClientDependencies: List[ModuleID] = List( "org.typelevel" %% "cats-effect" % catsEffectV, @@ -544,8 +518,6 @@ object Dependencies { "co.fs2" %% "fs2-io" % fs2V ) ++ scalacheckDependencies - val bcsBackendDependencies: List[ModuleID] = commonDependencies ++ refinedTypeDependenciesList ++ aliyunBatchComputeDependencies - val tesBackendDependencies: List[ModuleID] = akkaHttpDependencies val sfsBackendDependencies = List ( @@ -578,7 +550,6 @@ object Dependencies { val allProjectDependencies: List[ModuleID] = backendDependencies ++ - bcsBackendDependencies ++ centaurCwlRunnerDependencies ++ centaurDependencies ++ cloudSupportDependencies ++ @@ -589,7 +560,6 @@ object Dependencies { cwlDependencies ++ databaseMigrationDependencies ++ databaseSqlDependencies ++ - dockerHashingDependencies ++ draft2LanguageFactoryDependencies ++ drsLocalizerDependencies ++ engineDependencies ++ @@ -598,7 +568,6 @@ object Dependencies { implDrsDependencies ++ implFtpDependencies ++ languageFactoryDependencies ++ - ossFileSystemDependencies ++ perfDependencies ++ serverDependencies ++ sfsBackendDependencies ++ diff --git a/src/ci/bin/testCentaurBcs.sh b/src/ci/bin/testCentaurBcs.sh deleted file mode 100755 index 270da840432..00000000000 --- a/src/ci/bin/testCentaurBcs.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -export CROMWELL_BUILD_REQUIRES_SECURE=true -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test_bcs.inc.sh" || source test_bcs.inc.sh - -cromwell::build::setup_common_environment - -cromwell::build::bcs::setup_bcs_environment - -cromwell::build::setup_centaur_environment - -cromwell::build::assemble_jars - -# Instead of excluding tests, only include a fixed list of tests. This is because due to the -# numerous issues below, contributors did not like having to constantly update the exclude lists. -# https://github.com/broadinstitute/cromwell/issues/3522 -# https://github.com/broadinstitute/cromwell/issues/3523 -# https://github.com/broadinstitute/cromwell/issues/3524 -# https://github.com/broadinstitute/cromwell/issues/3518 -# https://github.com/broadinstitute/cromwell/issues/3519 -include_tests=( \ - -i abort.instant_abort \ - -i abort.sub_workflow_abort \ - -i aliased_subworkflows \ - -i array_io \ - -i array_literal_locations \ - -i arrays_scatters_ifs \ - -i bad_docker_name \ - -i bad_workflow_failure_mode \ - -i cacheBetweenWF \ - -i cacheWithinWF \ - -i chainfail \ - -i complex_types_files \ - -i composedenginefunctions \ - -i cwl_input_binding_expression \ - -i cwl_optionals \ - -i declarations \ - -i declarations_as_nodes \ - -i declarations_in_ifs \ - -i default_runtime_attributes \ - -i defined_function \ - -i dont_strip_line_prefix \ - -i dot_dir_stuck_running \ - -i draft3_declaration_chain \ - -i draft3_default_input_overrides \ - -i draft3_empty \ - -i draft3_import_structs \ - -i draft3_lots_of_nesting \ - -i draft3_nested_scatter \ - -i draft3_nested_struct \ - -i draft3_passthrough_value \ - -i draft3_sizeenginefunction \ - -i draft3_struct_output \ - -i draft3_taskless_engine_functions \ - -i empty_scatter \ - -i empty_string \ - -i exit \ - -i expression_lib_cwl \ - -i failures.terminal_status \ - -i filearrayoutput \ - -i floating_tags \ - -i forkjoin \ - -i hello_cwl \ - -i if_then_else_expressions \ - -i ifs_upstream_and_downstream \ - -i input_mirror \ - -i invalid_inputs_json \ - -i invalid_labels \ - -i invalid_options_json \ - -i invalid_runtime_attributes \ - -i invalid_wdl \ - -i length \ - # -i long_cmd \ # 2019-08-05 consistently timing out trying to read a < 100KB file in 60 seconds - -i lots_of_nesting \ - -i member_access \ - -i missing_imports \ - -i missing_sub_inputs \ - -i multiline_command_line \ - -i multiplesourcedarray \ - -i nested_lookups \ - -i null_input_values \ - -i object_access \ - -i optional_declarations \ - -i optional_parameter \ - -i output_filename_interpolation \ - -i output_redirection \ - -i passingfiles \ - -i prefix \ - -i public_http_import \ - -i readFromCacheFalse \ - -i read_tsv \ - -i read_write_json \ - -i read_write_map \ - -i referencingpreviousinputsandoutputs \ - -i runtime_attribute_expressions \ - -i runtime_failOnStderr \ - -i scatterchain \ - -i scattergather \ - -i scatters_in_ifs \ - -i select_functions \ - -i simple_if \ - -i simple_if_workflow_outputs \ - -i single_to_array_coercion \ - -i sizeenginefunction \ - -i square \ - -i stdout_stderr_passing \ - -i string_interpolation \ - -i sub_function \ - -i sub_workflow_decls \ - -i sub_workflow_hello_world \ - -i sub_workflow_interactions \ - -i sub_workflow_interactions_scatter \ - -i sub_workflow_no_output \ - -i sub_workflow_var_refs \ - -i subdirectory \ - -i subworkflows_in_ifs \ - -i taskless_engine_functions \ - -i test_file_outputs_from_input \ - -i three_step__subwf_cwl \ - -i unexpected_call_input_failure \ - -i unexpected_subworkflow_call_input_failure \ - -i valid_labels \ - -i variable_scoping \ - -i wdl_function_locations \ - -i workflow_output_declarations \ - -i workflow_type_and_version_default \ - -i workflow_type_and_version_wdl \ - -i workflowenginefunctions \ - -i writeToCache \ - -i write_lines \ - -i write_lines_files \ - -i write_tsv \ -) - -cromwell::build::run_centaur \ - -p 100 \ - -t 1m \ - -e localdockertest \ - "${include_tests[@]}" \ - -cromwell::build::generate_code_coverage diff --git a/src/ci/bin/test_bcs.inc.sh b/src/ci/bin/test_bcs.inc.sh deleted file mode 100644 index a415d0a49d2..00000000000 --- a/src/ci/bin/test_bcs.inc.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh - -# A set of common BCS functions for use in other scripts. -# -# Functions: -# -# - cromwell::build::bcs::* -# Functions for use in other BCS scripts -# -# - cromwell::private::bcs::* -# Functions for use only within this file by cromwell::build::bcs::* functions -# - -cromwell::private::bcs::bcs_install() { - cromwell::build::pip_install batchcompute-cli==1.7.1 --upgrade -} - -cromwell::private::bcs::bcs_run() { - local bcs_command - local bcs_command_result - - bcs_command="${1:?bcs_run called without a command}"; shift - bcs_command_result="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${bcs_command}_result.txt" - - # Login failures print out the access key so send output to a file. - bcs "${bcs_command}" "$@" &1 | tee "${bcs_command_result}" - - # bcs commands always exit with zero. make sure the result text does not contain "error". - if grep -q -i error "${bcs_command_result}"; then - echo "bcs ${bcs_command} failed" >&2 - grep -i error "${bcs_command_result}" >&2 - return 1 - else - return 0 - fi -} - -cromwell::private::bcs::try_bcs_login() { - local bcs_login_include - bcs_login_include="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/bcs_login.inc.sh" - if [[ -f "${bcs_login_include}" ]]; then - # shellcheck source=/dev/null - source "${bcs_login_include}" - fi -} - -cromwell::private::bcs::try_bcs_create_cluster() { - local cluster_name - - cluster_name=$(echo "cromwell_build_${CROMWELL_BUILD_PROVIDER}_${CROMWELL_BUILD_NUMBER}" | tr -c a-zA-Z0-9_- _) - - echo "Creating BCS cluster name: '${cluster_name}'" - - CROMWELL_BUILD_BCS_CLUSTER_ID=$( \ - cromwell::private::bcs::bcs_run \ - create_cluster \ - "${cluster_name}" \ - --image img-ubuntu \ - --type ecs.sn1ne.large \ - --nodes 8 \ - --vpc_cidr_block 192.168.1.0/24 \ - --no_cache_support \ - | grep 'Cluster created:' \ - | awk '{print $NF}' \ - ) - - echo "Created BCS cluster id: '${CROMWELL_BUILD_BCS_CLUSTER_ID}'" - - export CROMWELL_BUILD_BCS_CLUSTER_ID -} - -cromwell::private::bcs::try_bcs_delete_cluster() { - cromwell::private::bcs::bcs_run delete_cluster --yes "${CROMWELL_BUILD_BCS_CLUSTER_ID}" -} - -cromwell::private::bcs::bcs_login() { - cromwell::build::exec_retry_function cromwell::private::bcs::try_bcs_login -} - -cromwell::private::bcs::bcs_config() { - cromwell::private::bcs::bcs_run config --god true -} - -cromwell::private::bcs::bcs_create_cluster() { - cromwell::build::exec_retry_function cromwell::private::bcs::try_bcs_create_cluster - cromwell::build::add_exit_function cromwell::private::bcs::bcs_delete_cluster -} - -cromwell::private::bcs::bcs_delete_cluster() { - if [[ -n "${CROMWELL_BUILD_BCS_CLUSTER_ID}" ]]; then - cromwell::build::exec_retry_function cromwell::private::bcs::try_bcs_delete_cluster || true - fi -} - -cromwell::private::bcs::bcs_delete_old_resources() { - # Clean up assuming that all BCS jobs and clusters that are older than 3 hours are orphans to be deleted. jq 1.5 - # date functions all use UTC. https://stedolan.github.io/jq/manual/v1.5/#Dates Set the timezone environment variable - # to UTC before running the command just in case this script/bcs are run on a different zone outside of UTC. - echo "Please wait, removing old jobs…" - - TZ=utc \ - bcs job --all --show_json \ - | jq \ - -L "${CROMWELL_BUILD_RESOURCES_DIRECTORY}" \ - --raw-output 'include "bcs"; printIdsMoreThanSecondsOld(.; 3 * 60 * 60)' \ - | xargs -n 1 -I '{}' bash -c 'bcs delete_job --yes {} || true' - - echo "Please wait, removing old clusters…" - TZ=utc \ - bcs cluster --show_json \ - | jq \ - -L "${CROMWELL_BUILD_RESOURCES_DIRECTORY}" \ - --raw-output 'include "bcs"; printIdsMoreThanSecondsOld(.; 3 * 60 * 60)' \ - | xargs -n 1 -I '{}' bash -c 'bcs delete_cluster --yes {} || true' -} - -cromwell::build::bcs::setup_bcs_environment() { - cromwell::private::bcs::bcs_install - cromwell::build::exec_silent_function cromwell::private::bcs::bcs_login - cromwell::private::bcs::bcs_config - cromwell::private::bcs::bcs_delete_old_resources - - # Create the BCS cluster before sbt assembly as cluster creation takes a few minutes - cromwell::private::bcs::bcs_create_cluster -} diff --git a/src/ci/docker-compose/docker-compose-horicromtal.yml b/src/ci/docker-compose/docker-compose-horicromtal.yml index fcdbda8d432..bc2a20a4a91 100644 --- a/src/ci/docker-compose/docker-compose-horicromtal.yml +++ b/src/ci/docker-compose/docker-compose-horicromtal.yml @@ -21,7 +21,6 @@ services: -Dsystem.max-workflow-launch-count=0 -Dsystem.new-workflow-poll-rate=999999 -Dservices.MetadataService.config.metadata-summary-refresh-interval=Inf - - CROMWELL_BUILD_BCS_CLUSTER_ID - CROMWELL_BUILD_CENTAUR_256_BITS_KEY - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER - CROMWELL_BUILD_CENTAUR_JDBC_URL @@ -56,7 +55,6 @@ services: JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_CENTAUR_MANAGED_CONFIG} -Dwebservice.port=8000 -Dsystem.cromwell_id=summarizer - - CROMWELL_BUILD_BCS_CLUSTER_ID - CROMWELL_BUILD_CENTAUR_256_BITS_KEY - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER - CROMWELL_BUILD_CENTAUR_JDBC_URL @@ -99,7 +97,6 @@ services: -Dwebservice.port=${CROMWELL_BUILD_CENTAUR_MANAGED_PORT} -Dsystem.cromwell_id=frontend -Dservices.MetadataService.config.metadata-summary-refresh-interval=Inf - - CROMWELL_BUILD_BCS_CLUSTER_ID - CROMWELL_BUILD_CENTAUR_256_BITS_KEY - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER - CROMWELL_BUILD_CENTAUR_JDBC_URL diff --git a/src/ci/resources/bcs.jq b/src/ci/resources/bcs.jq deleted file mode 100644 index ef76bc3a628..00000000000 --- a/src/ci/resources/bcs.jq +++ /dev/null @@ -1,31 +0,0 @@ -# Convert a bcs date to an jq compatible date. -# The bcs date should already be in UTC. -# https://stedolan.github.io/jq/manual/v1.5/#Dates -def bcsToEpochSeconds(bcs_date): - bcs_date - | sub(" "; "T") - | sub("\\..*$"; "Z") - | fromdateiso8601; - -# Returns true if the jq function `now` is more than `seconds` ahead of `epoch_date_seconds`. -# https://stedolan.github.io/jq/manual/v1.5/#Dates -def isMoreThanSecondsOld(epoch_date_seconds; seconds): - now - epoch_date_seconds > seconds; - -# Filters the bcs date `key` if it is more than `seconds` old. -# https://stedolan.github.io/jq/manual/v1.5/#select(boolean_expression) -def filterMoreThanSecondsOld(key; seconds): - map(select(isMoreThanSecondsOld( - bcsToEpochSeconds(key); - seconds - ))); - -# Returns items under `key` that were created more than `seconds` ago. -# For bcs jobs and clusters the key is usually `.`. -# Expects `key` to be able to parse `{ Items: [ .Items[] | {Id, CreationTime} ] }` -def printIdsMoreThanSecondsOld(key; seconds): - key - | .Items - | filterMoreThanSecondsOld(.CreationTime; seconds) - | .[] - | .Id; diff --git a/src/ci/resources/bcs_application.conf.ctmpl b/src/ci/resources/bcs_application.conf.ctmpl deleted file mode 100644 index 16440f70634..00000000000 --- a/src/ci/resources/bcs_application.conf.ctmpl +++ /dev/null @@ -1,81 +0,0 @@ -include required(classpath("application.conf")) -include "build_application.inc.conf" - -{{with $cromwellBcs := secret (printf "secret/dsde/cromwell/common/cromwell-bcs")}} -backend { - default = "BCS" - - providers { - BCS { - actor-factory = "cromwell.backend.impl.bcs.BcsBackendLifecycleActorFactory" - - config { - root = "oss://cloud-cromwell-dev-self-cleaning/cromwell-dir" - region = "us-east-1" - access-id = "{{$cromwellBcs.Data.access_id}}" - access-key = "{{$cromwellBcs.Data.access_key}}" - - concurrent-job-limit = 50 - - filesystems { - oss { - auth { - endpoint = "oss-us-east-1.aliyuncs.com" - access-id = "{{$cromwellBcs.Data.access_id}}" - access-key = "{{$cromwellBcs.Data.access_key}}" - } - } - } - - default-runtime-attributes { - # Not 100% sure of the instance types, but as of April 2018 according to heshan.lhs@alibaba-inc.com the BCS - # itself needs some compute resources on the spun up VM. So we're using medium instances. - # - https://www.alibabacloud.com/help/doc-detail/25378.htm - # TODO: Is there an even smaller/faster image that we can use for BCS - #cluster: "OnDemand ecs.sn1.medium img-ubuntu" - # Alternatively leave a fixed cluster spun up via: - # bcs cc cromwell_test_cluster -i img-ubuntu -t ecs.sn1.medium -n 1 -d 'cromwell test cluster' - cluster: "Error: BA-6546 The environment variable CROMWELL_BUILD_BCS_CLUSTER_ID must be set/export pointing to a valid cluster id" - cluster: ${?CROMWELL_BUILD_BCS_CLUSTER_ID} - - # TODO: We should continue to allow users and our CI tests to cache images in their own OSS bucket - # BUT we should also be able to check the hash of the image in OSS using the config supplied credentials - # - https://www.alibabacloud.com/help/doc-detail/50452.htm?spm=a3c0i.l31815en.a3.109.50db5139VKk1FK - # - https://www.alibabacloud.com/product/oss?spm=a3c0i.7911826.1023975.dproductb1.454c737bJknGYt#resources - # Downloading from DockerHub is incredibly slow in Hangzhou. However for portability of WDL, we should use a - # new BCS runtime attribute for a `ossDockerRegistry` instead of this format? - # docker: "ubuntu/latest oss://broad-test/registry/ubuntu/" - - # If we do allow the above then we should NOT be trying to hash against DockerHub for the BCS backend. - # We should test if we can ping the OSS registry using our existing client code. - # TODO: https://github.com/broadinstitute/cromwell/issues/3518 For now, ignore docker. - ignoreDocker: true - - timeout: 3000 # None of our test workflow calls should be running longer than 3000 seconds - # Saw heshan.lhs@alibaba-inc.com set this. Not sure how it is used internally / if it is necessary - vpc: "192.168.1.0/24" - - # TODO: Embed the worker as a compiled resource - # TODO: Include the python as source code and not in the tar - workerPath: ${user.dir}/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/worker.tar.gz - } - } - - # Have the engine authenticate to docker.io. See BT-141 for more info. - include "dockerhub_provider_config_v1.inc.conf" - } - } -} - -engine { - filesystems { - oss { - auth { - endpoint = "oss-us-east-1.aliyuncs.com" - access-id = "{{$cromwellBcs.Data.access_id}}" - access-key = "{{$cromwellBcs.Data.access_key}}" - } - } - } -} -{{end}} diff --git a/src/ci/resources/bcs_login.inc.sh.ctmpl b/src/ci/resources/bcs_login.inc.sh.ctmpl deleted file mode 100644 index b3366087027..00000000000 --- a/src/ci/resources/bcs_login.inc.sh.ctmpl +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Used to login bcs from the command line. If/when there is a better way to bcs login please update testCentaurBcs.sh -# and remove all traces of this include. - -set -o errexit -o nounset -o pipefail - -local bcs_access_id -local bcs_access_key - -{{with $cromwellBcs := secret (printf "secret/dsde/cromwell/common/cromwell-bcs")}} -bcs_access_id="{{$cromwellBcs.Data.access_id}}" -bcs_access_key="{{$cromwellBcs.Data.access_key}}" -{{end}} - -cromwell::build::exec_silent_function \ - cromwell::private::bcs::bcs_run login us-east-1 "${bcs_access_id}" "${bcs_access_key}" >/dev/null diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsAsyncBackendJobExecutionActor.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsAsyncBackendJobExecutionActor.scala deleted file mode 100644 index 73d622eaecd..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsAsyncBackendJobExecutionActor.scala +++ /dev/null @@ -1,424 +0,0 @@ -package cromwell.backend.impl.bcs - -import better.files.File.OpenOptions -import com.aliyuncs.batchcompute.main.v20151111.BatchComputeClient -import com.aliyuncs.exceptions.{ClientException, ServerException} -import common.collections.EnhancedCollections._ -import common.util.StringUtil._ -import cromwell.backend._ -import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle} -import cromwell.backend.impl.bcs.RunStatus.{Finished, TerminalRunStatus} -import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob} -import cromwell.core.path.{DefaultPathBuilder, Path, PathFactory} -import cromwell.core.retry.SimpleExponentialBackoff -import cromwell.core.ExecutionEvent -import cromwell.filesystems.oss.OssPath -import wom.callable.Callable.OutputDefinition -import wom.callable.RuntimeEnvironment -import wom.core.FullyQualifiedName -import wom.expression.NoIoFunctionSet -import wom.types.WomSingleFileType -import wom.values._ -import mouse.all._ - -import scala.annotation.nowarn -import scala.concurrent.Future -import scala.concurrent.duration._ -import scala.util.{Success, Try} - -object BcsAsyncBackendJobExecutionActor { - val JobIdKey = "__bcs_job_id" -} - -final class BcsAsyncBackendJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams) - extends BackendJobLifecycleActor with StandardAsyncExecutionActor with BcsJobCachingActorHelper { - - type BcsPendingExecutionHandle = PendingExecutionHandle[StandardAsyncJob, BcsJob, RunStatus] - - override type StandardAsyncRunInfo = BcsJob - - override type StandardAsyncRunState = RunStatus - - def statusEquivalentTo(thiz: StandardAsyncRunState)(that: StandardAsyncRunState): Boolean = thiz == that - - override lazy val pollBackOff = SimpleExponentialBackoff(1.second, 5.minutes, 1.1) - - override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff(3.seconds, 30.seconds, 1.1) - - // override lazy val dockerImageUsed: Option[String] = runtimeAttributes.docker map {docker => docker.image} - override lazy val dockerImageUsed: Option[String] = None - override lazy val commandDirectory: Path = BcsJobPaths.BcsCommandDirectory.resolve(bcsJobPaths.callExecutionRoot.pathWithoutScheme) - - private[bcs] lazy val userTag = runtimeAttributes.tag.getOrElse("cromwell") - private[bcs] lazy val jobName: String = - List(userTag, jobDescriptor.workflowDescriptor.id.shortString, jobDescriptor.taskCall.identifier.localName.value) - .mkString("_") - // Avoid "Name ... must only contain characters within [a-zA-Z0-9_-] and not start with [0-9]." - .replaceAll("[^a-zA-Z0-9_-]", "_") - - override lazy val jobTag: String = jobDescriptor.key.tag - - private lazy val bcsWorkflowInputMount: BcsMount = bcsWorkflowPaths.getWorkflowInputMounts - private lazy val userDefinedMounts: List[BcsMount] = runtimeAttributes.mounts.toList.flatten :+ bcsWorkflowInputMount - // TODO: With a bit of refactoring this mutable var can be converted to a def or lazy val - private var inputMounts: List[BcsMount] = List.empty - - private[bcs] def ossPathToMount(ossPath: OssPath): BcsInputMount = { - val tmp = DefaultPathBuilder.get("/" + ossPath.pathWithoutScheme) - val dir = tmp.getParent - val local = BcsJobPaths.BcsTempInputDirectory.resolve(dir.pathAsString.md5SumShort).resolve(tmp.getFileName) - val ret = BcsInputMount(Left(ossPath), Left(local), writeSupport = false) - if (!inputMounts.exists(mount => mount.src == Left(ossPath) && mount.dest == Left(local))) { - inputMounts :+= ret - } - - ret - } - - private[bcs] def womFileToMount(file: WomFile): Option[BcsInputMount] = file match { - case path if userDefinedMounts exists(bcsMount => path.valueString.startsWith(BcsMount.toString(bcsMount.src))) => None - case path => PathFactory.buildPath(path.valueString, initializationData.pathBuilders) match { - case ossPath: OssPath => Some(ossPathToMount(ossPath)) - case _ => None - } - } - - private def bcsInputsFromWomFiles(prefix: String, - remotePathArray: Seq[WomFile], - jobDescriptor: BackendJobDescriptor): Iterable[BcsInputMount] = { - remotePathArray flatMap { remotePath => womFileToMount(remotePath) match { - case Some(mount) => Seq(mount) - case None => Seq.empty - } - } - } - - private[bcs] def getInputFiles(jobDescriptor: BackendJobDescriptor): Map[FullyQualifiedName, Seq[WomFile]] = { - val writeFunctionFiles = instantiatedCommand.createdFiles map { f => f.file.value.md5SumShort -> Seq(f.file) } - - val writeFunctionInputs = writeFunctionFiles map { - case (name, files) => name -> files - } - - // Collect all WomFiles from inputs to the call. - val callInputFiles: Map[FullyQualifiedName, Seq[WomFile]] = jobDescriptor.fullyQualifiedInputs safeMapValues { - _.collectAsSeq { case w: WomFile => w } - } - - callInputFiles ++ writeFunctionInputs - } - - private[bcs] def generateBcsInputs(jobDescriptor: BackendJobDescriptor): Unit = { - val _ = getInputFiles(jobDescriptor) flatMap { - case (name, files) => bcsInputsFromWomFiles(name, files, jobDescriptor) - } - } - - private def relativePath(path: String): Path = { - val absolutePath = DefaultPathBuilder.get(path) match { - case p if !p.isAbsolute => commandDirectory.resolve(p) - case p => p - } - - absolutePath - } - - private[bcs] lazy val callRawOutputFiles: List[WomFile] = { - import cats.syntax.validated._ - def evaluateFiles(output: OutputDefinition): List[WomFile] = { - Try ( - output.expression.evaluateFiles(jobDescriptor.localInputs, NoIoFunctionSet, output.womType).map(_.toList map { _.file }) - ).getOrElse(List.empty[WomFile].validNel) - .getOrElse(List.empty) - } - - // val womFileOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, PureStandardLibraryFunctions) - - jobDescriptor.taskCall.callable.outputs.flatMap(evaluateFiles) - } - - private[bcs] def isOutputOssFileString(s: String): Boolean = { - callRawOutputFiles.exists({ - case file: WomSingleFile if file.value == s => true - case _ => false - }) - } - - private[bcs] def generateBcsOutputs(jobDescriptor: BackendJobDescriptor): Seq[BcsMount] = { - callRawOutputFiles.flatMap(_.flattenFiles).distinct flatMap { womFile => - womFile match { - case singleFile: WomSingleFile => List(generateBcsSingleFileOutput(singleFile)) - case globFile: WomGlobFile => generateBcsGlobFileOutputs(globFile) - case unlistedDirectory: WomUnlistedDirectory => generateUnlistedDirectoryOutputs(unlistedDirectory) - } - } - } - - private def generateBcsSingleFileOutput(wdlFile: WomSingleFile): BcsOutputMount = { - val destination = getPath(wdlFile.valueString) match { - case Success(ossPath: OssPath) => ossPath - case Success(path: Path) if !path.isAbsolute => relativeOutputPath(path) - case _ => callRootPath.resolve(wdlFile.value.stripPrefix("/")) - } - - val src = relativePath(wdlFile.valueString) - - BcsOutputMount(Left(src), Left(destination), writeSupport = false) - } - - protected def generateBcsGlobFileOutputs(womFile: WomGlobFile): List[BcsOutputMount] = { - val globName = GlobFunctions.globName(womFile.value) - val globDirectory = globName + "/" - val globListFile = globName + ".list" - val bcsGlobDirectoryDestinationPath = callRootPath.resolve(globDirectory) - val bcsGlobListFileDestinationPath = callRootPath.resolve(globListFile) - - // We need both the glob directory and the glob list: - List( - BcsOutputMount(Left(relativePath(globDirectory)), Left(bcsGlobDirectoryDestinationPath), writeSupport = false), - BcsOutputMount(Left(relativePath(globListFile)), Left(bcsGlobListFileDestinationPath), writeSupport = false) - ) - } - - private def generateUnlistedDirectoryOutputs(womFile: WomUnlistedDirectory): List[BcsOutputMount] = { - val directoryPath = womFile.value.ensureSlashed - val directoryListFile = womFile.value.ensureUnslashed + ".list" - val bcsDirDestinationPath = callRootPath.resolve(directoryPath) - val bcsListDestinationPath = callRootPath.resolve(directoryListFile) - - // We need both the collection directory and the collection list: - List( - BcsOutputMount(Left(relativePath(directoryPath)), Left(bcsDirDestinationPath), writeSupport = false), - BcsOutputMount(Left(relativePath(directoryListFile)), Left(bcsListDestinationPath), writeSupport = false) - ) - } - - private[bcs] def getOssFileName(ossPath: OssPath): String = { - getPath(ossPath.pathWithoutScheme) match { - case Success(path) => path.getFileName.pathAsString - case _ => ossPath.pathWithoutScheme - } - } - - private[bcs] def localizeOssPath(ossPath: OssPath): String = { - if (isOutputOssFileString(ossPath.pathAsString) && !ossPath.isAbsolute) { - if (ossPath.exists) { - ossPathToMount(ossPath).dest match { - case Left(p) => p.normalize().pathAsString - case _ => throw new RuntimeException("only support oss") - } - } else { - commandDirectory.resolve(getOssFileName(ossPath)).normalize().pathAsString - } - } else { - userDefinedMounts collectFirst { - case bcsMount: BcsMount if ossPath.pathAsString.startsWith(BcsMount.toString(bcsMount.src)) => - bcsMount.dest match { - case Left(p) => p.resolve(ossPath.pathAsString.stripPrefix(BcsMount.toString(bcsMount.src))).pathAsString - case _ => throw new RuntimeException("only support oss") - } - } getOrElse { - val mount = ossPathToMount(ossPath) - BcsMount.toString(mount.dest) - } - } - } - - private[bcs] def relativeOutputPath(path: Path): Path = { - if (isOutputOssFileString(path.pathAsString)) { - bcsJobPaths.callRoot.resolve(path.pathAsString).normalize() - } else { - path - } - } - - private[bcs] def mapWomFile(womFile: WomFile): WomFile = { - getPath(womFile.valueString) match { - case Success(ossPath: OssPath) => - WomFile(WomSingleFileType, localizeOssPath(ossPath)) - case Success(path: Path) if !path.isAbsolute => - WomFile(WomSingleFileType, relativeOutputPath(path).pathAsString) - case _ => womFile - } - } - - override def preProcessWomFile(womFile: WomFile): WomFile = mapWomFile(womFile) - - override def mapCommandLineWomFile(womFile: WomFile): WomFile = mapWomFile(womFile) - - override def runtimeEnvironmentPathMapper(env: RuntimeEnvironment): RuntimeEnvironment = { - def localize(path: String): String = (WomSingleFile(path) |> mapRuntimeEnvs).valueString - env.copy(outputPath = env.outputPath |> localize, tempPath = env.tempPath |> localize) - } - - private[bcs] def mapRuntimeEnvs(womFile: WomSingleFile): WomFile = { - getPath(womFile.valueString) match { - case Success(ossPath: OssPath) => - WomFile(WomSingleFileType, BcsJobPaths.BcsCommandDirectory.resolve(ossPath.pathWithoutScheme).pathAsString) - case _ => womFile - } - - } - - override def isTerminal(runStatus: RunStatus): Boolean = { - runStatus match { - case _ : TerminalRunStatus => true - case _ => false - } - } - - override def getTerminalEvents(runStatus: RunStatus): Seq[ExecutionEvent] = { - runStatus match { - case successStatus: Finished => successStatus.eventList - case unknown => - throw new RuntimeException(s"handleExecutionSuccess not called with RunStatus.Success. Instead got $unknown") - } - } - - override def handleExecutionFailure(runStatus: RunStatus, - returnCode: Option[Int]): Future[ExecutionHandle] = { - runStatus match { - case RunStatus.Failed(jobId, Some(errorMessage), _) => - val exception = new Exception(s"Job id $jobId failed: '$errorMessage'") - Future.successful(FailedNonRetryableExecutionHandle(exception, returnCode, None)) - case _ => super.handleExecutionFailure(runStatus, returnCode) - } - } - - override def isDone(runStatus: RunStatus): Boolean = { - runStatus match { - case _: Finished => - runtimeAttributes.autoReleaseJob match { - case Some(true) | None => - bcsClient.deleteJob(runStatus.jobId) - case _ => - } - true - case _ => false - } - } - - private[bcs] lazy val rcBcsOutput = BcsOutputMount( - Left(commandDirectory.resolve(bcsJobPaths.returnCodeFilename)), Left(bcsJobPaths.returnCode), writeSupport = false) - - private[bcs] lazy val stdoutBcsOutput = BcsOutputMount( - Left(commandDirectory.resolve(bcsJobPaths.defaultStdoutFilename)), Left(standardPaths.output), writeSupport = false) - private[bcs] lazy val stderrBcsOutput = BcsOutputMount( - Left(commandDirectory.resolve(bcsJobPaths.defaultStderrFilename)), Left(standardPaths.error), writeSupport = false) - - private[bcs] lazy val uploadBcsWorkerPackage = { - bcsJobPaths.workerPath.writeByteArray(BcsJobCachingActorHelper.workerScript.getBytes)(OpenOptions.default) - - bcsJobPaths.workerPath - } - - @nowarn("msg=a type was inferred to be `Object`; this may indicate a programming error.") - override def executeAsync(): Future[ExecutionHandle] = { - commandScriptContents.fold( - errors => Future.failed(new RuntimeException(errors.toList.mkString(", "))), - bcsJobPaths.script.write) - - - setBcsVerbose() - - val envs = bcsEnvs - - val bcsJob = new BcsJob( - jobName, - jobTag, - bcsCommandLine, - uploadBcsWorkerPackage, - bcsMounts, - envs, - runtimeAttributes, - Some(bcsJobPaths.bcsStdoutPath), - Some(bcsJobPaths.bcsStderrPath), - bcsClient) - - for { - jobId <- Future.fromTry(bcsJob.submit()) - } yield PendingExecutionHandle(jobDescriptor, StandardAsyncJob(jobId), Option(bcsJob), previousState = None) - } - - override def recoverAsync(jobId: StandardAsyncJob) = executeAsync() - - override def pollStatusAsync(handle: BcsPendingExecutionHandle): Future[RunStatus] = { - val jobId = handle.pendingJob.jobId - val bcsJob: BcsJob = handle.runInfo.getOrElse(throw new RuntimeException("empty run job info ")) - - Future.fromTry(bcsJob.getStatus(jobId)) - } - - override def mapOutputWomFile(wdlFile: WomFile): WomFile = { - wdlFileToOssPath(generateBcsOutputs(jobDescriptor))(wdlFile) - } - - private[bcs] def wdlFileToOssPath(bcsOutputs: Seq[BcsMount])(wdlFile: WomFile): WomFile = { - bcsOutputs collectFirst { - case bcsOutput if BcsMount.toString(bcsOutput.src).endsWith(wdlFile.valueString) => WomFile(WomSingleFileType, BcsMount.toString(bcsOutput.dest)) - } getOrElse wdlFile - } - - override def tryAbort(job: StandardAsyncJob): Unit = { - for { - client <- Try(initializationData.bcsConfiguration.bcsClient getOrElse(throw new RuntimeException("empty run job info "))) - resp <- Try(client.getJob(job.jobId)) - status <- RunStatusFactory.getStatus(job.jobId, resp.getJob.getState) - } yield { - status match { - case _: RunStatus.TerminalRunStatus => - for { - _ <- Try(client.deleteJob(job.jobId)) - } yield job - case _ => - for { - _ <- Try(client.stopJob(job.jobId)) - _ <- Try(client.deleteJob(job.jobId)) - } yield job - } - } - () - } - - override def isFatal(throwable: Throwable): Boolean = super.isFatal(throwable) || isFatalBcsException(throwable) - - private[bcs] def isFatalBcsException(throwable: Throwable) = { - throwable match { - case e: ClientException if e.getErrCode.startsWith("Invalid") => true - case _ => false - } - } - - override def isTransient(throwable: Throwable): Boolean = { - throwable match { - case _: ServerException => true - case e: ClientException if e.getErrCode == "InternalError" => true - case e: ClientException if e.getErrCode.startsWith("Throttling") => true - case _ => false - } - } - - private[bcs] def setBcsVerbose(): Unit = { - runtimeAttributes.verbose match { - case Some(verbose) => BatchComputeClient.verbose = verbose - case None => BatchComputeClient.verbose = false - } - } - - private[bcs] lazy val bcsEnvs: Map[String, String] = { - val mount = ossPathToMount(bcsJobPaths.script.asInstanceOf[OssPath]) - - Map( - BcsJobPaths.BcsEnvCwdKey -> commandDirectory.pathAsString, - BcsJobPaths.BcsEnvExecKey -> BcsMount.toString(mount.dest), - BcsJobPaths.BcsEnvStdoutKey -> commandDirectory.resolve(bcsJobPaths.defaultStdoutFilename).pathAsString, - BcsJobPaths.BcsEnvStderrKey -> commandDirectory.resolve(bcsJobPaths.defaultStderrFilename).pathAsString - ) - } - - private[bcs] lazy val bcsMounts: Seq[BcsMount] ={ - generateBcsInputs(jobDescriptor) - runtimeAttributes.mounts.getOrElse(Seq.empty) ++ inputMounts ++ - generateBcsOutputs(jobDescriptor) :+ rcBcsOutput :+ stdoutBcsOutput :+ stderrBcsOutput :+ bcsWorkflowInputMount - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsBackendInitializationData.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsBackendInitializationData.scala deleted file mode 100644 index 89a97b98f90..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsBackendInitializationData.scala +++ /dev/null @@ -1,14 +0,0 @@ -package cromwell.backend.impl.bcs - -import cromwell.backend.io.WorkflowPaths -import cromwell.backend.standard.{StandardInitializationData, StandardValidatedRuntimeAttributesBuilder} -import cromwell.core.path.PathBuilder - -final case class BcsBackendInitializationData -( - override val workflowPaths: WorkflowPaths, - override val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder, - bcsConfiguration: BcsConfiguration, - pathBuilders: List[PathBuilder] -) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[BcsExpressionFunctions]) - diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsBackendLifecycleActorFactory.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsBackendLifecycleActorFactory.scala deleted file mode 100644 index 053b3b4cadb..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsBackendLifecycleActorFactory.scala +++ /dev/null @@ -1,41 +0,0 @@ -package cromwell.backend.impl.bcs - -import akka.actor.ActorRef -import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} -import cromwell.backend.standard._ -import cromwell.backend.BackendInitializationData -import cromwell.backend.impl.bcs.callcaching.BcsBackendCacheHitCopyingActor -import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActor -import wom.graph.CommandCallNode - -import scala.util.{Success, Try} - - -final case class BcsBackendLifecycleActorFactory(val name: String, val configurationDescriptor: BackendConfigurationDescriptor) - extends StandardLifecycleActorFactory { - override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[BcsInitializationActor] - override lazy val asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] = classOf[BcsAsyncBackendJobExecutionActor] - - override def jobIdKey: String = BcsAsyncBackendJobExecutionActor.JobIdKey - - val bcsConfiguration = new BcsConfiguration(configurationDescriptor) - - override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], serviceRegistryActor: ActorRef, restarting: Boolean): StandardInitializationActorParams = { - BcsInitializationActorParams(workflowDescriptor, calls, bcsConfiguration, serviceRegistryActor) - } - - override lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = { - Option(classOf[BcsBackendCacheHitCopyingActor]) - } - - override def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, initializationData: Option[BackendInitializationData]) = { - Try(BackendInitializationData.as[BcsBackendInitializationData](initializationData)) match { - case Success(bcsData) => - bcsData.bcsConfiguration.dockerHashEndpoint match { - case Some(endpoint) => List(bcsData.bcsConfiguration.dockerCredentials, Option(endpoint)).flatten - case None => List(bcsData.bcsConfiguration.dockerCredentials).flatten - } - case _ => List.empty[Any] - } - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsClusterIdOrConfiguration.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsClusterIdOrConfiguration.scala deleted file mode 100644 index 047af4dabf2..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsClusterIdOrConfiguration.scala +++ /dev/null @@ -1,56 +0,0 @@ -package cromwell.backend.impl.bcs - - -import scala.util.{Try, Success, Failure} -import scala.util.matching.Regex - -final case class AutoClusterConfiguration(resourceType: String, - instanceType: String, - imageId: String, - spotStrategy: Option[String] = None, - spotPriceLimit: Option[Float] = None, - clusterId: Option[String] = None) - - -object BcsClusterIdOrConfiguration { - type BcsClusterIdOrConfiguration = Either[String, AutoClusterConfiguration] - - val idPattern: Regex = """(cls-[^\s]+)""".r - val resourceTypePattern = """(OnDemand|Spot)""".r - val defaultResourceType = "OnDemand" - - val imageIdPattern = """([^\s]+)""".r - - val spotStrategyPattern = """(SpotWithPriceLimit|SpotAsPriceGo)""".r - - val spotPriceLimitPattern = """([01]\.\d{1,3})""".r - - // no suitable default instance type availabe - val instanceTypePattern = """([be]cs[^\s]+)""".r - - val instanceAndImagePattern = s"""$instanceTypePattern\\s+$imageIdPattern""".r - - val resourceAndInstanceAndImagePattern = s"""$resourceTypePattern\\s+$instanceTypePattern\\s+$imageIdPattern""".r - - val spotPattern = s"""$resourceAndInstanceAndImagePattern\\s+$spotStrategyPattern\\s+$spotPriceLimitPattern""".r - - val attachClusterSimplePattern = s"""$instanceAndImagePattern\\s+$idPattern""".r - - val attachClusterPattern = s"""$resourceAndInstanceAndImagePattern\\s+$idPattern""".r - - val attachClusterSpotPattern = s"""$spotPattern\\s+$idPattern""".r - - - def parse(cluster: String): Try[BcsClusterIdOrConfiguration] = { - cluster match { - case idPattern(clusterId) => Success(Left(clusterId)) - case instanceAndImagePattern(instanceType, imageId) => Success(Right(AutoClusterConfiguration(defaultResourceType, instanceType, imageId))) - case attachClusterSimplePattern(instanceType, imageId, clusterId) =>Success(Right(AutoClusterConfiguration(defaultResourceType, instanceType, imageId, clusterId=Option(clusterId)))) - case resourceAndInstanceAndImagePattern(resourceType, instanceType, imageId) => Success(Right(AutoClusterConfiguration(resourceType, instanceType, imageId))) - case attachClusterPattern(resourceType, instanceType, imageId, clusterId) => Success(Right(AutoClusterConfiguration(resourceType, instanceType, imageId, clusterId = Option(clusterId)))) - case spotPattern(resourceType, instanceType, imageId, spotStrategy, spotPriceLimit) => Success(Right(AutoClusterConfiguration(resourceType, instanceType, imageId, Option(spotStrategy), Option(spotPriceLimit.toFloat)))) - case attachClusterSpotPattern(resourceType, instanceType, imageId, spotStrategy, spotPriceLimit, clusterId) => Success(Right(AutoClusterConfiguration(resourceType, instanceType, imageId, Option(spotStrategy), Option(spotPriceLimit.toFloat), Option(clusterId)))) - case _ => Failure(new IllegalArgumentException("must be some string like 'cls-xxxx' or 'OnDemand ecs.s1.large img-ubuntu' or 'OnDemand ecs.s1.large img-ubuntu cls-xxxx'")) - } - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsConfiguration.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsConfiguration.scala deleted file mode 100644 index c3119079211..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsConfiguration.scala +++ /dev/null @@ -1,71 +0,0 @@ -package cromwell.backend.impl.bcs - -import com.aliyuncs.auth.BasicCredentials -import com.aliyuncs.batchcompute.main.v20151111.BatchComputeClient -import cromwell.backend.BackendConfigurationDescriptor -import net.ceedubs.ficus.Ficus._ -import cromwell.backend.impl.bcs.callcaching.{CopyCachedOutputs, UseOriginalCachedOutputs} -import cromwell.core.DockerConfiguration - -object BcsConfiguration{ - val OssEndpointKey = "ossEndpoint" - val OssIdKey = "ossId" - val OssSecretKey = "ossSecret" - val OssTokenKey = "ossToken" -} - -final class BcsConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) { - val runtimeConfig = configurationDescriptor.backendRuntimeAttributesConfig - val bcsRegion: Option[String] = configurationDescriptor.backendConfig.as[Option[String]]("region") - - val bcsUserDefinedRegion: Option[String] = configurationDescriptor.backendConfig.as[Option[String]]("user-defined-region") - - val bcsUserDefinedDomain: Option[String] = configurationDescriptor.backendConfig.as[Option[String]]("user-defined-domain") - - val bcsAccessId: Option[String] = configurationDescriptor.backendConfig.as[Option[String]]("access-id") - - val bcsAccessKey: Option[String] = configurationDescriptor.backendConfig.as[Option[String]]("access-key") - - val bcsSecurityToken: Option[String] = configurationDescriptor.backendConfig.as[Option[String]]("security-token") - - val ossEndpoint = configurationDescriptor.backendConfig.as[Option[String]]("filesystems.oss.auth.endpoint").getOrElse("") - val ossAccessId = configurationDescriptor.backendConfig.as[Option[String]]("filesystems.oss.auth.access-id").getOrElse("") - val ossAccessKey = configurationDescriptor.backendConfig.as[Option[String]]("filesystems.oss.auth.access-key").getOrElse("") - val ossSecurityToken = configurationDescriptor.backendConfig.as[Option[String]]("filesystems.oss.auth.security-token").getOrElse("") - - val duplicationStrategy = { - configurationDescriptor.backendConfig.as[Option[String]]("filesystems.oss.caching.duplication-strategy").getOrElse("reference") match { - case "copy" => CopyCachedOutputs - case "reference" => UseOriginalCachedOutputs - case other => throw new IllegalArgumentException(s"Unrecognized caching duplication strategy: $other. Supported strategies are copy and reference. See reference.conf for more details.") - } - } - - lazy val dockerHashAccessId = DockerConfiguration.dockerHashLookupConfig.as[Option[String]]("alibabacloudcr.auth.access-id") - lazy val dockerHashAccessKey = DockerConfiguration.dockerHashLookupConfig.as[Option[String]]("alibabacloudcr.auth.access-key") - lazy val dockerHashSecurityToken = DockerConfiguration.dockerHashLookupConfig.as[Option[String]]("alibabacloudcr.auth.security-token") - lazy val dockerHashEndpoint = DockerConfiguration.dockerHashLookupConfig.as[Option[String]]("alibabacloudcr.auth.endpoint") - - val dockerCredentials = { - for { - id <- dockerHashAccessId - key <- dockerHashAccessKey - } yield new BasicCredentials(id, key) - } - - val bcsClient: Option[BatchComputeClient] = { - val userDefinedRegion = for { - region <- bcsUserDefinedRegion - domain <- bcsUserDefinedDomain - } yield { - BatchComputeClient.addEndpoint(region, domain) - region - } - - for { - region <- userDefinedRegion orElse bcsRegion - id <- bcsAccessId - key <- bcsAccessKey - } yield new BatchComputeClient(region, id, key) - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsDisk.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsDisk.scala deleted file mode 100644 index d2d52e2521c..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsDisk.scala +++ /dev/null @@ -1,24 +0,0 @@ -package cromwell.backend.impl.bcs - -import scala.util.{Try, Success, Failure} - -trait BcsDisk { - val diskType: String - val sizeInGB: Int -} - -final case class BcsSystemDisk(diskType: String, sizeInGB: Int) extends BcsDisk -final case class BcsDataDisk(diskType: String, sizeInGB: Int, mountPoint: String) extends BcsDisk - -object BcsDisk{ - val systemDiskPattern = s"""(\\S+)\\s+(\\d+)""".r - val dataDiskPattern = s"""(\\S+)\\s+(\\d+)\\s+(\\S+)""".r - - def parse(s: String): Try[BcsDisk] = { - s match { - case systemDiskPattern(diskType, sizeInGB) => Success(BcsSystemDisk(diskType, sizeInGB.toInt)) - case dataDiskPattern(diskType, sizeInGB, mountPoint) => Success(BcsDataDisk(diskType, sizeInGB.toInt, mountPoint)) - case _ => Failure(new IllegalArgumentException("disk must be 'cloud 40' or 'cloud 200 /home/input/'")) - } - } -} \ No newline at end of file diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsDocker.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsDocker.scala deleted file mode 100644 index 91250956b26..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsDocker.scala +++ /dev/null @@ -1,24 +0,0 @@ -package cromwell.backend.impl.bcs - -import scala.util.{Failure, Success, Try} - -trait BcsDocker { - val image: String -} - -final case class BcsDockerWithoutPath(image: String) extends BcsDocker -final case class BcsDockerWithPath(image: String, path: String) extends BcsDocker - - -object BcsDocker{ - val dockerWithPathPattern = s"""(\\S+)\\s+(\\S+)""".r - val dockerWithoutPathPatter = s"""(\\S+)""".r - - def parse(s: String): Try[BcsDocker] = { - s match { - case dockerWithoutPathPatter(dockerImage) => Success(BcsDockerWithoutPath(dockerImage)) - case dockerWithPathPattern(dockerImage, dockerPath) => Success(BcsDockerWithPath(dockerImage, dockerPath)) - case _ => Failure(new IllegalArgumentException("must be 'ubuntu/latest oss://docker-reg/ubuntu/'")) - } - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsExpressionFunctions.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsExpressionFunctions.scala deleted file mode 100644 index 0e47bf9faa1..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsExpressionFunctions.scala +++ /dev/null @@ -1,42 +0,0 @@ -package cromwell.backend.impl.bcs - -import common.validation.ErrorOr.ErrorOr -import cromwell.backend.BackendJobDescriptor -import cromwell.backend.standard.{StandardExpressionFunctions, StandardExpressionFunctionsParams} -import cromwell.filesystems.oss.OssPathBuilder -import cromwell.filesystems.oss.OssPathBuilder.{InvalidOssPath, PossiblyValidRelativeOssPath, ValidFullOssPath} -import wom.graph.CommandCallNode -import wom.values.WomGlobFile - -import scala.concurrent.Future - -final case class BcsExpressionFunctions(override val standardParams: StandardExpressionFunctionsParams) - extends StandardExpressionFunctions(standardParams) { - - override def preMapping(str: String) = { - OssPathBuilder.validateOssPath(str) match { - case _: ValidFullOssPath => str - case PossiblyValidRelativeOssPath => callContext.root.resolve(str.stripPrefix("/")).pathAsString - case invalid: InvalidOssPath => throw new IllegalArgumentException(invalid.errorMessage) - } - } - - // TODO: BCS: When globs are supported this override should be removed. - // https://github.com/broadinstitute/cromwell/issues/3519 - // See usages of cwl.CommandLineTool.CwlOutputJson - override def glob(pattern: String): Future[Seq[String]] = { - if (pattern == "cwl.output.json") { - Future.successful(Nil) - } else { - super.glob(pattern) - } - } - - // TODO: BCS: When globs are supported this override should be removed. - // https://github.com/broadinstitute/cromwell/issues/3519 - // See usages of cwl.CommandLineTool.CwlOutputJson - override def findGlobOutputs(call: CommandCallNode, jobDescriptor: BackendJobDescriptor): ErrorOr[List[WomGlobFile]] = { - val base = super.findGlobOutputs(call, jobDescriptor) - base.map(_.filterNot(_.value == "cwl.output.json")) - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsInitializationActor.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsInitializationActor.scala deleted file mode 100644 index d93b7076420..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsInitializationActor.scala +++ /dev/null @@ -1,45 +0,0 @@ -package cromwell.backend.impl.bcs - -import akka.actor.ActorRef -import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder} -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor} -import cromwell.core.path.PathBuilder - -import scala.concurrent.Future -import wom.graph.CommandCallNode - - -final case class BcsInitializationActorParams -( - workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[CommandCallNode], - bcsConfiguration: BcsConfiguration, - serviceRegistryActor: ActorRef -) extends StandardInitializationActorParams { - override val configurationDescriptor: BackendConfigurationDescriptor = bcsConfiguration.configurationDescriptor -} - -final class BcsInitializationActor(params: BcsInitializationActorParams) - extends StandardInitializationActor(params) { - - private val bcsConfiguration = params.bcsConfiguration - - override lazy val pathBuilders: Future[List[PathBuilder]] = - standardParams.configurationDescriptor.pathBuildersWithDefault(workflowDescriptor.workflowOptions) - - override lazy val workflowPaths: Future[BcsWorkflowPaths] = pathBuilders map { - BcsWorkflowPaths(workflowDescriptor, bcsConfiguration.configurationDescriptor.backendConfig, _) - } - - override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = - BcsRuntimeAttributes.runtimeAttributesBuilder(bcsConfiguration.runtimeConfig) - - override def beforeAll(): Future[Option[BackendInitializationData]] = { - pathBuilders map { builders => BcsMount.pathBuilders = builders} - - for { - paths <- workflowPaths - builders <- pathBuilders - } yield Option(BcsBackendInitializationData(paths, runtimeAttributesBuilder, bcsConfiguration, builders)) - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJob.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJob.scala deleted file mode 100644 index c19e1228864..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJob.scala +++ /dev/null @@ -1,226 +0,0 @@ -package cromwell.backend.impl.bcs - -import com.aliyuncs.batchcompute.main.v20151111.BatchComputeClient -import com.aliyuncs.batchcompute.model.v20151111._ -import com.aliyuncs.batchcompute.pojo.v20151111._ -import cromwell.core.ExecutionEvent -import cromwell.core.path.Path - -import scala.jdk.CollectionConverters._ -import scala.util.{Failure, Success, Try} - -object BcsJob{ - val BcsDockerImageEnvKey = "BATCH_COMPUTE_DOCKER_IMAGE" - val BcsDockerPathEnvKey = "BATCH_COMPUTE_DOCKER_REGISTRY_OSS_PATH" -} - -final case class BcsJob(name: String, - description: String, - commandString: String, - packagePath: Path, - mounts: Seq[BcsMount], - envs: Map[String, String], - runtime: BcsRuntimeAttributes, - stdoutPath: Option[Path], - stderrPath: Option[Path], - batchCompute: BatchComputeClient) { - - lazy val lazyDisks = new Disks - lazy val lazyConfigs = new Configs - lazy val lazyVpc = new VPC - lazy val lazyTask = new TaskDescription - lazy val lazyJob = new JobDescription - lazy val lazyCmd = new Command - - def submit(): Try[String] = Try{ - val request: CreateJobRequest = new CreateJobRequest - request.setJobDescription(jobDesc) - val response: CreateJobResponse = batchCompute.createJob(request) - val jobId = response.getJobId - jobId - } - - def getStatus(jobId: String): Try[RunStatus] = Try{ - val request: GetJobRequest = new GetJobRequest - request.setJobId(jobId) - val response: GetJobResponse = batchCompute.getJob(request) - val job = response.getJob - val status = job.getState - val message = job.getMessage - val eventList = Seq[ExecutionEvent]() - RunStatusFactory.getStatus(jobId, status, Some(message), Some(eventList)) match { - case Success(status) => status - case Failure(e) => throw e - } - } - - def cancel(jobId: String): Unit = { - // XXX: Do nothing currently. - } - - private[bcs] def systemDisk: Option[SystemDisk] = runtime.systemDisk map { disk => - val systemDisk = new SystemDisk() - systemDisk.setType(disk.diskType) - systemDisk.setSize(disk.sizeInGB) - systemDisk - } - - private[bcs] def dataDisk: Option[DataDisk] = runtime.dataDisk map { disk => - val dataDisk = new DataDisk - dataDisk.setType(disk.diskType) - dataDisk.setSize(disk.sizeInGB) - dataDisk.setMountPoint(disk.mountPoint) - dataDisk - } - - // XXX: maybe more elegant way to reduce two options? - private[bcs] def disks: Option[Disks] = { - (systemDisk, dataDisk) match { - case (Some(sys), Some(data)) => - lazyDisks.setSystemDisk(sys) - lazyDisks.setDataDisk(data) - Some(lazyDisks) - case (Some(sys), None) => - lazyDisks.setSystemDisk(sys) - Some(lazyDisks) - case (None, Some(data)) => - lazyDisks.setDataDisk(data) - Some(lazyDisks) - case (None, None) => None - } - } - - private[bcs] def vpc: Option[VPC] = { - (runtime.vpc flatMap {v => v.cidrBlock}, runtime.vpc flatMap {v => v.vpcId}) match { - case (Some(cidr), Some(id)) => - lazyVpc.setCidrBlock(cidr) - lazyVpc.setVpcId(id) - Some(lazyVpc) - case (Some(cidr), None) => - lazyVpc.setCidrBlock(cidr) - Some(lazyVpc) - case (None, Some(vpc)) => - lazyVpc.setVpcId(vpc) - Some(lazyVpc) - case (None, None) => None - } - } - - private[bcs] def configs: Option[Configs] = { - (vpc, disks) match { - case (Some(bcsVpc), Some(bcsDisks)) => - lazyConfigs.setDisks(bcsDisks) - val networks = new Networks - networks.setVpc(bcsVpc) - lazyConfigs.setNetworks(networks) - Some(lazyConfigs) - case (Some(bcsVpc), None) => - val networks = new Networks - networks.setVpc(bcsVpc) - lazyConfigs.setNetworks(networks) - Some(lazyConfigs) - case (None, Some(bcsDisks)) => - lazyConfigs.setDisks(bcsDisks) - Some(lazyConfigs) - case (None, None) => None - } - } - - private[bcs] def params: Parameters = { - val parames = new Parameters - lazyCmd.setPackagePath(packagePath.pathAsString) - lazyCmd.setEnvVars(environments.asJava) - lazyCmd.setCommandLine(commandString) - - dockers foreach {docker => lazyCmd.setDocker(docker)} - stdoutPath foreach {path => parames.setStdoutRedirectPath(path.normalize().pathAsString + "/")} - stderrPath foreach {path => parames.setStderrRedirectPath(path.normalize().pathAsString + "/")} - - parames.setCommand(lazyCmd) - parames - } - - private[bcs] def environments: Map[String, String] = { - runtime.docker match { - case None => - runtime.dockerTag match { - case Some(docker: BcsDockerWithoutPath) => envs + (BcsJob.BcsDockerImageEnvKey -> docker.image) - case Some(docker: BcsDockerWithPath) => envs + (BcsJob.BcsDockerPathEnvKey -> docker.path) + (BcsJob.BcsDockerImageEnvKey -> docker.image) - case _ => envs - } - case _ => envs - } - } - - val dockers: Option[Command.Docker] = { - runtime.docker match { - case Some(docker: BcsDockerWithoutPath) => - val dockers = new Command.Docker - dockers.setImage(docker.image) - Some(dockers) - case _ => None - } - } - - private[bcs] def jobDesc: JobDescription = { - lazyJob.setName(name) - lazyJob.setDescription(description) - lazyJob.setType("DAG") - - val dag = new DAG - dag.addTask("cromwell", taskDesc) - lazyJob.setDag(dag) - - // NOTE: Do NOT set auto release here or we will not be able to get status after the job completes. - lazyJob.setAutoRelease(false) - - lazyJob - } - - private[bcs] def taskDesc: TaskDescription = { - lazyTask.setParameters(params) - lazyTask.setInstanceCount(1) - - runtime.timeout foreach {timeout => lazyTask.setTimeout(timeout.toLong)} - - val cluster = runtime.cluster getOrElse(throw new IllegalArgumentException("cluster id or auto cluster configuration is mandatory")) - cluster.fold(handleClusterId, handleAutoCluster) - - val mnts = new Mounts - mounts foreach { - case input: BcsInputMount => - mnts.addEntries(input.toBcsMountEntry) - case output: BcsOutputMount => - var srcStr = BcsMount.toString(output.src) - if (BcsMount.toString(output.dest).endsWith("/") && !srcStr.endsWith("/")) { - srcStr += "/" - } - lazyTask.addOutputMapping(srcStr, BcsMount.toString(output.dest)) - } - - lazyTask.setMounts(mnts) - - lazyTask - } - - private def handleAutoCluster(config: AutoClusterConfiguration): Unit = { - val autoCluster = new AutoCluster - autoCluster.setImageId(runtime.imageId.getOrElse(config.imageId)) - autoCluster.setInstanceType(config.instanceType) - autoCluster.setResourceType(config.resourceType) - - config.spotStrategy foreach {strategy => autoCluster.setSpotStrategy(strategy)} - config.spotPriceLimit foreach {priceLimit => autoCluster.setSpotPriceLimit(priceLimit)} - config.clusterId foreach {clusterId => autoCluster.setClusterId(clusterId)} - runtime.reserveOnFail foreach {reserve => autoCluster.setReserveOnFail(reserve)} - val userData = runtime.userData map {datas => Map(datas map {data => data.key -> data.value}: _*)} - userData foreach {datas => autoCluster.setUserData(datas.asJava)} - - configs foreach (bcsConfigs => autoCluster.setConfigs(bcsConfigs)) - runtime.isv foreach(isv => autoCluster.setDependencyIsvService(isv)) - - lazyTask.setAutoCluster(autoCluster) - } - - private def handleClusterId(clusterId: String): Unit = lazyTask.setClusterId(clusterId) -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJobCachingActorHelper.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJobCachingActorHelper.scala deleted file mode 100644 index e4aa2413f73..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJobCachingActorHelper.scala +++ /dev/null @@ -1,52 +0,0 @@ -package cromwell.backend.impl.bcs - -import akka.actor.Actor -import cromwell.backend.standard.StandardCachingActorHelper -import cromwell.core.logging.JobLogging -import cromwell.core.path.Path - -object BcsJobCachingActorHelper { - val workerScript: String = - s"""|#!/bin/bash - |export script=$$cwd/$$(basename $$exec) - |export rc=$$cwd/rc - | - |( - |mkdir -p $$cwd - |cp -rf $$exec $$script - |cd $$cwd - |/bin/bash -c $$script - |) - """.stripMargin -} - -trait BcsJobCachingActorHelper extends StandardCachingActorHelper { - this: Actor with JobLogging => - - bcsWorkflowPaths.tag = runtimeAttributes.tag.getOrElse("") - - lazy val initializationData: BcsBackendInitializationData = { - backendInitializationDataAs[BcsBackendInitializationData] - } - - def bcsClient = initializationData.bcsConfiguration.bcsClient.getOrElse(throw new RuntimeException("no bcs client available")) - - lazy val bcsWorkflowPaths: BcsWorkflowPaths = workflowPaths.asInstanceOf[BcsWorkflowPaths] - - lazy val bcsJobPaths: BcsJobPaths = jobPaths.asInstanceOf[BcsJobPaths] - - lazy val bcsConfiguration: BcsConfiguration = initializationData.bcsConfiguration - - lazy val runtimeAttributes = BcsRuntimeAttributes(validatedRuntimeAttributes, bcsConfiguration.runtimeConfig) - - lazy val callRootPath: Path = bcsJobPaths.callExecutionRoot - - lazy val returnCodeFilename: String = bcsJobPaths.returnCodeFilename - lazy val returnCodeGcsPath: Path = bcsJobPaths.returnCode - lazy val standardPaths = bcsJobPaths.standardPaths - lazy val bcsStdoutFile: Path = standardPaths.output - lazy val bcsStderrFile: Path = standardPaths.error - - //lazy val bcsCommandLine = "bash -c $(pwd)/cromwell_bcs && sync" - lazy val bcsCommandLine = "./worker" -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJobPaths.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJobPaths.scala deleted file mode 100644 index 6353f51e3e1..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsJobPaths.scala +++ /dev/null @@ -1,29 +0,0 @@ -package cromwell.backend.impl.bcs - -import cromwell.backend.BackendJobDescriptorKey -import cromwell.backend.io.JobPaths -import cromwell.core.path.{DefaultPathBuilder, Path} - -object BcsJobPaths { - val BcsLogPathKey = "bcsLog" - val BcsEnvExecKey = "exec" - val BcsEnvCwdKey = "cwd" - val BcsEnvStdoutKey = "stdout" - val BcsEnvStderrKey = "stderr" - val BcsCommandDirectory: Path = DefaultPathBuilder.get("/cromwell_root") - val BcsTempInputDirectory: Path = DefaultPathBuilder.get("/cromwell_inputs") - val BcsStdoutRedirectPath = "bcs-stdout" - val BcsStderrRedirectPath = "bcs-stderr" -} - -final case class BcsJobPaths(workflowPaths: BcsWorkflowPaths, jobKey: BackendJobDescriptorKey, override val isCallCacheCopyAttempt: Boolean = false) extends JobPaths { - - import BcsJobPaths._ - - val workerFileName = "worker" - val workerPath = callRoot.resolve(workerFileName) - val bcsStdoutPath = callRoot.resolve(BcsStdoutRedirectPath) - val bcsStderrPath = callRoot.resolve(BcsStderrRedirectPath) - - override def forCallCacheCopyAttempts: JobPaths = this.copy(isCallCacheCopyAttempt = true) -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsMount.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsMount.scala deleted file mode 100644 index 757405d23fb..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsMount.scala +++ /dev/null @@ -1,127 +0,0 @@ -package cromwell.backend.impl.bcs - -import cats.data.Validated._ -import cats.syntax.apply._ -import cats.syntax.validated._ -import com.aliyuncs.batchcompute.pojo.v20151111.MountEntry -import common.exception.MessageAggregation -import common.validation.ErrorOr._ -import cromwell.backend.impl.bcs.BcsMount.PathType -import cromwell.core.path.{Path, PathBuilder, PathFactory} - -import scala.util.{Success, Try} -import scala.util.matching.Regex - -object BcsMount { - type PathType = Either[Path, String] - - def toString(p: PathType): String = { - p match { - case Left(p) => - p.pathAsString - case Right(s) => - return s - } - } - - val supportFileSystemTypes = List("oss", "nas", "smb", "lustre").mkString("|") - - var pathBuilders: List[PathBuilder] = List() - - val remotePrefix = s"""(?:$supportFileSystemTypes)""" + """://[^\s]+""" - val localPath = """/[^\s]+""" - val writeSupport = """true|false""" - - val inputMountPattern: Regex = s"""($remotePrefix)\\s+($localPath)\\s+($writeSupport)""".r - val outputMountPattern: Regex = s"""($localPath)\\s+($remotePrefix)\\s+($writeSupport)""".r - - def parse(s: String): Try[BcsMount] = { - val validation: ErrorOr[BcsMount] = s match { - case inputMountPattern(remote, local, writeSupport) => - (validateRemote(remote), validateLocal(remote, local), validateBoolean(writeSupport)) mapN { (src, dest, ws) => new BcsInputMount(src, dest, ws)} - case outputMountPattern(local, oss, writeSupport) => - (validateLocal(oss, local), validateRemote(oss), validateBoolean(writeSupport)) mapN { (src, dest, ws) => new BcsOutputMount(src, dest, ws)} - case _ => s"Mount strings should be of the format 'oss://my-bucket/inputs/ /home/inputs/ true' or '/home/outputs/ oss://my-bucket/outputs/ false'".invalidNel - } - - Try(validation match { - case Valid(mount) => mount - case Invalid(nels) => - throw new UnsupportedOperationException with MessageAggregation { - val exceptionContext = "" - val errorMessages: List[String] = nels.toList - } - }) - } - - private def validateRemote(value: String): ErrorOr[PathType] = { - Try(PathFactory.buildPath(value, pathBuilders)) match { - case Success(p) => - Left(p).validNel - case _ => - Right(value).validNel - } - } - private def validateLocal(remote: String, local: String): ErrorOr[PathType] = { - if (remote.endsWith("/") == local.endsWith("/")) { - Try(PathFactory.buildPath(local, pathBuilders)) match { - case Success(p) => - Left(p).validNel - case _=> - Right(local).validNel - } - } else { - "oss and local path type not match".invalidNel - } - } - - private def validateBoolean(value: String): ErrorOr[Boolean] = { - try { - value.toBoolean.validNel - } catch { - case _: IllegalArgumentException => s"$value not convertible to a Boolean".invalidNel - } - } -} - -trait BcsMount { - import BcsMount._ - var src: PathType - var dest: PathType - var writeSupport: Boolean - - def toBcsMountEntry: MountEntry -} - -final case class BcsInputMount(var src: PathType, var dest: PathType, var writeSupport: Boolean) extends BcsMount { - def toBcsMountEntry: MountEntry = { - var destStr = BcsMount.toString(dest) - if (BcsMount.toString(src).endsWith("/") && !destStr.endsWith("/")) { - destStr += "/" - } - - val entry = new MountEntry - entry.setSource(BcsMount.toString(src)) - entry.setDestination(destStr) - entry.setWriteSupport(writeSupport) - - entry - } - -} -final case class BcsOutputMount(var src: PathType, var dest: PathType, var writeSupport: Boolean) extends BcsMount { - def toBcsMountEntry: MountEntry = { - var srcStr = BcsMount.toString(src) - if (BcsMount.toString(dest).endsWith("/") && !srcStr.endsWith("/")) { - srcStr += "/" - } - - - val entry = new MountEntry - entry.setSource(srcStr) - entry.setDestination(BcsMount.toString(dest)) - entry.setWriteSupport(writeSupport) - - entry - } -} \ No newline at end of file diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsRuntimeAttributes.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsRuntimeAttributes.scala deleted file mode 100644 index da73c3ef747..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsRuntimeAttributes.scala +++ /dev/null @@ -1,394 +0,0 @@ -package cromwell.backend.impl.bcs - -import cats.data.Validated._ -import cats.syntax.apply._ -import cats.syntax.validated._ -import com.typesafe.config.Config -import common.validation.ErrorOr._ -import cromwell.backend.impl.bcs.BcsClusterIdOrConfiguration.BcsClusterIdOrConfiguration -import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder -import cromwell.backend.validation._ -import net.ceedubs.ficus.Ficus._ -import wom.types._ -import wom.values._ - -import scala.util.{Failure, Success, Try} - - -trait OptionalWithDefault[A] { - this: RuntimeAttributesValidation[A] => - protected val config: Option[Config] - - override protected def staticDefaultOption: Option[WomValue] = { - Try(this.configDefaultWomValue(config)) match { - case Success(value: Option[WomValue]) => value - case Failure(_) => None - } - } -} - -final case class BcsRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, - dockerTag: Option[BcsDocker], - docker: Option[BcsDocker], - failOnStderr: Boolean, - mounts: Option[Seq[BcsMount]], - userData: Option[Seq[BcsUserData]], - cluster: Option[BcsClusterIdOrConfiguration], - imageId: Option[String], - systemDisk: Option[BcsSystemDisk], - dataDisk: Option[BcsDataDisk], - reserveOnFail: Option[Boolean], - autoReleaseJob: Option[Boolean], - timeout: Option[Int], - verbose: Option[Boolean], - vpc: Option[BcsVpcConfiguration], - tag: Option[String], - isv:Option[String]) - -object BcsRuntimeAttributes { - - val MountsKey = "mounts" - val UserDataKey = "userData" - val MountsDefaultValue = WomString("") - val ReserveOnFailKey = "reserveOnFail" - val ReserveOnFailDefault = false - val AutoReleaseJobKey = "autoReleaseJob" - val AutoReleaseJobDefault = WomBoolean(true) - val TimeoutKey = "timeout" - val TimeoutDefault = WomInteger(21600) - val VerboseKey = "verbose" - val ClusterKey = "cluster" - val DockerKey = "docker" - val SystemDiskKey = "systemDisk" - val DataDiskKey = "dataDisk" - val VpcKey = "vpc" - val TagKey = "tag" - - private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig) - - private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - - private def clusterValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[BcsClusterIdOrConfiguration] = ClusterValidation.optionalWithDefault(runtimeConfig) - - private def dockerTagValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[BcsDocker] = DockerTagValidation.optionalWithDefault(runtimeConfig) - private def dockerValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[BcsDocker] = DockerValidation.optionalWithDefault(runtimeConfig) - - private def userDataValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Seq[BcsUserData]] = UserDataValidation.optionalWithDefault(runtimeConfig) - - private def systemDiskValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[BcsSystemDisk] = SystemDiskValidation.optionalWithDefault(runtimeConfig) - private def dataDiskValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[BcsDataDisk] = DataDiskValidation.optionalWithDefault(runtimeConfig) - - private def reserveOnFailValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Boolean] = ReserveOnFailValidation.optionalWithDefault(runtimeConfig) - - private def autoReleaseJobValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Boolean] = AutoReleaseJobValidation.optionalWithDefault(runtimeConfig) - - private def mountsValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Seq[BcsMount]] = MountsValidation.optionalWithDefault(runtimeConfig) - - private def timeoutValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Int] = TimeoutValidation.optionalWithDefault(runtimeConfig) - - private def verboseValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Boolean] = VerboseValidation.optionalWithDefault(runtimeConfig) - - private def vpcValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[BcsVpcConfiguration] = VpcValidation.optionalWithDefault(runtimeConfig) - - private def tagValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[String] = TagValidation.optionalWithDefault(runtimeConfig) - - private def imageIdValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[String] = ImageIdValidation.optionalWithDefault(runtimeConfig) - - private def isvValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[String] = IsvValidation.optionalWithDefault(runtimeConfig) - - def runtimeAttributesBuilder(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = { - val defaults = StandardValidatedRuntimeAttributesBuilder.default(backendRuntimeConfig).withValidation( - mountsValidation(backendRuntimeConfig), - userDataValidation(backendRuntimeConfig), - clusterValidation(backendRuntimeConfig), - systemDiskValidation(backendRuntimeConfig), - dataDiskValidation(backendRuntimeConfig), - reserveOnFailValidation(backendRuntimeConfig), - autoReleaseJobValidation(backendRuntimeConfig), - timeoutValidation(backendRuntimeConfig), - verboseValidation(backendRuntimeConfig), - vpcValidation(backendRuntimeConfig), - tagValidation(backendRuntimeConfig), - imageIdValidation(backendRuntimeConfig), - isvValidation(backendRuntimeConfig), - ) - - // TODO: docker trips up centaur testing, for now https://github.com/broadinstitute/cromwell/issues/3518 - if (backendRuntimeConfig.exists(_.getOrElse("ignoreDocker", false))) { - defaults - } else { - defaults.withValidation( - dockerTagValidation(backendRuntimeConfig), - dockerValidation(backendRuntimeConfig) - ) - } - } - - def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes, backendRuntimeConfig: Option[Config]): BcsRuntimeAttributes = { - val failOnStderr: Boolean = - RuntimeAttributesValidation.extract(failOnStderrValidation(backendRuntimeConfig), validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = - RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), validatedRuntimeAttributes) - val mounts: Option[Seq[BcsMount]] = RuntimeAttributesValidation.extractOption(mountsValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val userData: Option[Seq[BcsUserData]] = RuntimeAttributesValidation.extractOption(userDataValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - - val cluster: Option[BcsClusterIdOrConfiguration] = RuntimeAttributesValidation.extractOption(clusterValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val imageId: Option[String] = RuntimeAttributesValidation.extractOption(imageIdValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val dockerTag: Option[BcsDocker] = RuntimeAttributesValidation.extractOption(dockerTagValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val docker: Option[BcsDocker] = RuntimeAttributesValidation.extractOption(dockerValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val systemDisk: Option[BcsSystemDisk] = RuntimeAttributesValidation.extractOption(systemDiskValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val dataDisk: Option[BcsDataDisk] = RuntimeAttributesValidation.extractOption(dataDiskValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - - val reserveOnFail: Option[Boolean] = RuntimeAttributesValidation.extractOption(reserveOnFailValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val autoReleaseJob: Option[Boolean] = RuntimeAttributesValidation.extractOption(autoReleaseJobValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val timeout: Option[Int] = RuntimeAttributesValidation.extractOption(timeoutValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val verbose: Option[Boolean] = RuntimeAttributesValidation.extractOption(verboseValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val vpc: Option[BcsVpcConfiguration] = RuntimeAttributesValidation.extractOption(vpcValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val tag: Option[String] = RuntimeAttributesValidation.extractOption(tagValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - val isv: Option[String] = RuntimeAttributesValidation.extractOption(isvValidation(backendRuntimeConfig).key, validatedRuntimeAttributes) - - new BcsRuntimeAttributes( - continueOnReturnCode, - dockerTag, - docker, - failOnStderr, - mounts, - userData, - cluster, - imageId, - systemDisk, - dataDisk, - reserveOnFail, - autoReleaseJob, - timeout, - verbose, - vpc, - tag, - isv - ) - } -} - -object MountsValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[Seq[BcsMount]] = new MountsValidation(config).optional -} - -class MountsValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[Seq[BcsMount]] with OptionalWithDefault[Seq[BcsMount]] { - override def key: String = BcsRuntimeAttributes.MountsKey - - override def coercion: Iterable[WomType] = Set(WomStringType, WomArrayType(WomStringType)) - - override protected def validateValue: PartialFunction[WomValue, ErrorOr[Seq[BcsMount]]] = { - case WomString(value) => validateMounts(value.split(",\\s*").toSeq) - case WomArray(wdlType, values) if wdlType.memberType == WomStringType => - validateMounts(values.map(_.valueString)) - } - - private def validateMounts(mounts: Seq[String]): ErrorOr[Seq[BcsMount]] = { - val mountNels: Seq[ErrorOr[BcsMount]] = mounts filter { s => !s.trim().isEmpty } map validateMounts - val sequenced: ErrorOr[Seq[BcsMount]] = sequenceNels(mountNels) - sequenced - } - - private def validateMounts(mount: String): ErrorOr[BcsMount] = { - BcsMount.parse(mount) match { - case scala.util.Success(mnt) => mnt.validNel - case scala.util.Failure(ex) => ex.getMessage.invalidNel - } - } - - private def sequenceNels(nels: Seq[ErrorOr[BcsMount]]): ErrorOr[Seq[BcsMount]] = { - val emptyMountNel: ErrorOr[Vector[BcsMount]] = Vector.empty[BcsMount].validNel - val mountsNel: ErrorOr[Vector[BcsMount]] = nels.foldLeft(emptyMountNel) { - (acc, v) => (acc, v) mapN { (a, v) => a :+ v } - } - mountsNel - } - - override protected def missingValueMessage: String = - s"Expecting $key runtime attribute to be a comma separated String or Array[String]" -} - -object UserDataValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[Seq[BcsUserData]] = new UserDataValidation(config).optional -} - -class UserDataValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[Seq[BcsUserData]] with OptionalWithDefault[Seq[BcsUserData]]{ - override def key: String = BcsRuntimeAttributes.UserDataKey - - override def usedInCallCaching: Boolean = true - - override def coercion: Iterable[WomType] = Set(WomStringType, WomArrayType(WomStringType)) - - override protected def validateValue: PartialFunction[WomValue, ErrorOr[Seq[BcsUserData]]] = { - case WomString(value) => validateUserData(value.split(",\\s*").toSeq) - case WomArray(wdlType, values) if wdlType.memberType == WomStringType => - validateUserData(values.map(_.valueString)) - } - - private def validateUserData(mounts: Seq[String]): ErrorOr[Seq[BcsUserData]] = { - val userDataNels: Seq[ErrorOr[BcsUserData]] = mounts filter { s => !s.trim().isEmpty } map validateUserData - val sequenced: ErrorOr[Seq[BcsUserData]] = sequenceNels(userDataNels) - sequenced - } - - private def validateUserData(data: String): ErrorOr[BcsUserData] = { - BcsUserData.parse(data) match { - case scala.util.Success(userData) => userData.validNel - case scala.util.Failure(ex) => ex.getMessage.invalidNel - } - } - - private def sequenceNels(nels: Seq[ErrorOr[BcsUserData]]): ErrorOr[Seq[BcsUserData]] = { - val emptyDataNel: ErrorOr[Vector[BcsUserData]] = Vector.empty[BcsUserData].validNel - val datasNel: ErrorOr[Vector[BcsUserData]] = nels.foldLeft(emptyDataNel) { - (acc, v) => (acc, v) mapN { (a, v) => a :+ v } - } - datasNel - } - - override protected def missingValueMessage: String = - s"Expecting $key runtime attribute to be a comma separated String or Array[String]" -} - -object ReserveOnFailValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[Boolean] = new ReserveOnFailValidation(config).optional -} - -class ReserveOnFailValidation(override val config: Option[Config]) extends BooleanRuntimeAttributesValidation(BcsRuntimeAttributes.ReserveOnFailKey) with OptionalWithDefault[Boolean] - -object AutoReleaseJobValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[Boolean] = new AutoReleaseJobValidation(config).optional -} - -class AutoReleaseJobValidation(override val config: Option[Config]) extends BooleanRuntimeAttributesValidation(BcsRuntimeAttributes.AutoReleaseJobKey) with OptionalWithDefault[Boolean] - -object TimeoutValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[Int] = new TimeoutValidation(config).optional -} - -class TimeoutValidation(override val config: Option[Config]) extends IntRuntimeAttributesValidation(BcsRuntimeAttributes.TimeoutKey) with OptionalWithDefault[Int] - -object VerboseValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[Boolean] = new VerboseValidation(config).optional -} - -class VerboseValidation(override val config: Option[Config]) extends BooleanRuntimeAttributesValidation(BcsRuntimeAttributes.VerboseKey) with OptionalWithDefault[Boolean] - - -object ClusterValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[BcsClusterIdOrConfiguration] = new ClusterValidation(config).optional -} - -class ClusterValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[BcsClusterIdOrConfiguration] with OptionalWithDefault[BcsClusterIdOrConfiguration] -{ - override def key: String = "cluster" - - override def coercion: Iterable[WomType] = Set(WomStringType) - - override def validateValue: PartialFunction[WomValue, ErrorOr[BcsClusterIdOrConfiguration]] = { - case WomString(s) => BcsClusterIdOrConfiguration.parse(s.toString) match { - case Success(cluster) => cluster.validNel - case Failure(t) => t.getMessage.invalidNel - } - } -} - -object SystemDiskValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[BcsSystemDisk] = new SystemDiskValidation(config).optional -} - -class SystemDiskValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[BcsSystemDisk] with OptionalWithDefault[BcsSystemDisk] -{ - override def key: String = "systemDisk" - override def coercion: Iterable[WomType] = Set(WomStringType) - override def validateValue: PartialFunction[WomValue, ErrorOr[BcsSystemDisk]] = { - case WomString(s) => BcsDisk.parse(s.toString) match { - case Success(disk: BcsSystemDisk) => disk.validNel - case _ => s"system disk should be string like 'cloud 40'".invalidNel - } - } -} - -object DataDiskValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[BcsDataDisk] = new DataDiskValidation(config).optional -} - -class DataDiskValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[BcsDataDisk] with OptionalWithDefault[BcsDataDisk] -{ - override def key: String = "dataDisk" - override def coercion: Iterable[WomType] = Set(WomStringType) - override def validateValue: PartialFunction[WomValue, ErrorOr[BcsDataDisk]] = { - case WomString(s) => BcsDisk.parse(s.toString) match { - case Success(disk: BcsDataDisk) => disk.validNel - case _ => s"system disk should be string like 'cloud 40 /home/data/'".invalidNel - } - } -} - -object DockerTagValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[BcsDocker] = new DockerTagValidation(config).optional -} - -class DockerTagValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[BcsDocker] with OptionalWithDefault[BcsDocker] -{ - override def key: String = "dockerTag" - override def coercion: Iterable[WomType] = Set(WomStringType) - override def validateValue: PartialFunction[WomValue, ErrorOr[BcsDocker]] = { - case WomString(s) => BcsDocker.parse(s.toString) match { - case Success(docker: BcsDocker) => docker.validNel - case _ => s"docker must be 'dockerImage dockerPath' like".invalidNel - } - } -} - -object DockerValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[BcsDocker] = new DockerValidation(config).optional -} - -class DockerValidation(override val config: Option[Config]) extends DockerTagValidation(config) -{ - override def key: String = "docker" - override def usedInCallCaching: Boolean = true -} - -object VpcValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[BcsVpcConfiguration] = new VpcValidation(config).optional -} - -class VpcValidation(override val config: Option[Config]) extends RuntimeAttributesValidation[BcsVpcConfiguration] with OptionalWithDefault[BcsVpcConfiguration] -{ - override def key: String = "vpc" - override def coercion: Iterable[WomType] = Set(WomStringType) - override def validateValue: PartialFunction[WomValue, ErrorOr[BcsVpcConfiguration]] = { - case WomString(s) => BcsVpcConfiguration.parse(s.toString) match { - case Success(vpc: BcsVpcConfiguration) => vpc.validNel - case _ => s"vpc must be '192.168.0.0/16 vpc-xxxx' like".invalidNel - } - } -} - -object TagValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[String] = new TagValidation(config).optional -} - -class TagValidation(override val config: Option[Config]) extends StringRuntimeAttributesValidation("tag") with OptionalWithDefault[String] - -object ImageIdValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[String] = new ImageIdValidation(config).optional -} - -class ImageIdValidation(override val config: Option[Config]) extends StringRuntimeAttributesValidation("imageId") with OptionalWithDefault[String] -{ - override def usedInCallCaching: Boolean = true -} - -object IsvValidation { - def optionalWithDefault(config: Option[Config]): OptionalRuntimeAttributesValidation[String] = new IsvValidation(config).optional -} - -class IsvValidation(override val config: Option[Config]) extends StringRuntimeAttributesValidation("isv") with OptionalWithDefault[String] -{ - override def usedInCallCaching: Boolean = true -} - diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsUserData.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsUserData.scala deleted file mode 100644 index 375ae943d49..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsUserData.scala +++ /dev/null @@ -1,43 +0,0 @@ -package cromwell.backend.impl.bcs - -import cats.data.Validated._ -import cats.syntax.apply._ -import common.validation.ErrorOr._ -import cats.syntax.validated._ -import common.exception.MessageAggregation - -import scala.util.Try -import scala.util.matching.Regex - -object BcsUserData { - val keyPattern = """[^\s]+""" - val valuePattern = """[^\s]+""" - val inputMountPattern: Regex = s"""($keyPattern)\\s+($valuePattern)""".r - - def parse(s: String): Try[BcsUserData] = { - val validation: ErrorOr[BcsUserData] = s match { - case inputMountPattern(key, value) => (validateKey(key), validateValue(value)) mapN ((k, v) => new BcsUserData(k, v)) - case _ => s"error user data entry".invalidNel - } - - Try(validation match { - case Valid(userData) => userData - case Invalid(nels) => - throw new UnsupportedOperationException with MessageAggregation { - val exceptionContext = "" - val errorMessages: List[String] = nels.toList - } - }) - } - - private def validateKey(key: String): ErrorOr[String] = { - key.validNel - } - - private def validateValue(value: String): ErrorOr[String] = { - value.validNel - } - -} - -final case class BcsUserData(key: String, value: String) diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsVpcConfiguration.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsVpcConfiguration.scala deleted file mode 100644 index eac121ad745..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsVpcConfiguration.scala +++ /dev/null @@ -1,25 +0,0 @@ -package cromwell.backend.impl.bcs - - -import scala.util.{Failure, Success, Try} - - -final case class BcsVpcConfiguration(cidrBlock: Option[String] = None, - vpcId: Option[String] = None) - - -object BcsVpcConfiguration { - val cidrBlockPattern = """(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]){1,2})""".r - val vpcIdPattern = """(vpc-[^\s]+)""".r - - def parse(s: String): Try[BcsVpcConfiguration] = { - val cidrBlock = cidrBlockPattern findFirstIn s - val vpcId = vpcIdPattern findFirstIn s - - if (cidrBlock.isEmpty && vpcId.isEmpty) { - Failure(new IllegalArgumentException("vpc configuration must be a string like '192.168.0.0/16 vpc-xxxx' ")) - } else { - Success(BcsVpcConfiguration(cidrBlock, vpcId)) - } - } -} \ No newline at end of file diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsWorkflowPaths.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsWorkflowPaths.scala deleted file mode 100644 index 1082acaa0f9..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/BcsWorkflowPaths.scala +++ /dev/null @@ -1,38 +0,0 @@ -package cromwell.backend.impl.bcs - -import com.typesafe.config.Config -import cromwell.backend.io.WorkflowPaths -import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} -import cromwell.core.path.{Path, PathBuilder} - -object BcsWorkflowPaths { - val WorkFlowTagKey = "bcs_workflow_tag" -} - -case class BcsWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescriptor, - override val config: Config, - override val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { - - import BcsWorkflowPaths._ - override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): BcsJobPaths = { - new BcsJobPaths(workflowPaths.asInstanceOf[BcsWorkflowPaths], jobKey) - } - - override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor) - - override protected def workflowPathBuilder(root: Path): Path = { - workflowDescriptor.breadCrumbs.foldLeft(root)((acc, breadCrumb) => { - breadCrumb.toPath(acc) - }).resolve(workflowDescriptor.callable.name).resolve(tag).resolve(workflowDescriptor.id.toString + "/") - } - - var tag: String = { - workflowDescriptor.workflowOptions.get(WorkFlowTagKey).getOrElse("") - } - - private[bcs] def getWorkflowInputMounts: BcsInputMount = { - val src = workflowRoot - val dest = BcsJobPaths.BcsTempInputDirectory.resolve(src.pathWithoutScheme) - BcsInputMount(Left(src), Left(dest), true) - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/RunStatus.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/RunStatus.scala deleted file mode 100644 index fbfa3149e0f..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/RunStatus.scala +++ /dev/null @@ -1,85 +0,0 @@ -package cromwell.backend.impl.bcs - -import cromwell.core.ExecutionEvent - -import scala.util.{Failure, Success, Try} - -sealed trait RunStatus { - import RunStatus._ - val jobId: String - val status: String - - def isTerminated: Boolean - - def isRunningOrComplete = this match { - case _: Running | _: TerminalRunStatus => true - case _ => false - } - - override def toString = status -} - -object RunStatus { - final case class Waiting(override val jobId: String) extends RunStatus { - override val status = "Waiting" - - override def isTerminated: Boolean = false - } - - final case class Running(override val jobId: String) extends RunStatus { - override val status = "Running" - - override def isTerminated: Boolean = false - } - - sealed trait TerminalRunStatus extends RunStatus { - def eventList: Seq[ExecutionEvent] - - override def isTerminated: Boolean = true - } - - sealed trait UnsuccessfulRunStatus extends TerminalRunStatus { - val errorMessage: Option[String] - lazy val prettyPrintedError: String = errorMessage map { e => s" Message: $e" } getOrElse "" - } - - final case class Finished(override val jobId: String, eventList: Seq[ExecutionEvent]) extends TerminalRunStatus { - override val status = "Finished" - } - - object UnsuccessfulRunStatus { - def apply(jobId: String, status: String, errorMessage: Option[String], eventList: Seq[ExecutionEvent]): UnsuccessfulRunStatus = { - if (status == "Stopped") { - Stopped(jobId, errorMessage, eventList) - } else { - Failed(jobId, errorMessage, eventList) - } - } - } - - final case class Failed(override val jobId: String, - errorMessage: Option[String], - eventList: Seq[ExecutionEvent]) extends UnsuccessfulRunStatus { - override val status = "Failed" - } - - final case class Stopped(override val jobId: String, - errorMessage: Option[String], - eventList: Seq[ExecutionEvent]) extends UnsuccessfulRunStatus { - override val status = "Stopped" - } -} - -object RunStatusFactory { - def getStatus(jobId: String, status: String, errorMessage: Option[String] = None, eventList: Option[Seq[ExecutionEvent]] = None): Try[RunStatus] = { - import RunStatus._ - status match { - case "Waiting" => Success(Waiting(jobId)) - case "Running" => Success(Running(jobId)) - case "Stopped" => Success(Stopped(jobId, errorMessage, eventList.getOrElse(Seq.empty))) - case "Failed" => Success(Failed(jobId, errorMessage, eventList.getOrElse(Seq.empty))) - case "Finished" => Success(Finished(jobId, eventList.getOrElse(Seq.empty))) - case _ => Failure(new RuntimeException(s"job {$jobId} turns to an invalid batchcompue status: {$status}")) - } - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/callcaching/BcsBackendCacheHitCopyingActor.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/callcaching/BcsBackendCacheHitCopyingActor.scala deleted file mode 100644 index 0285edf3e0a..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/callcaching/BcsBackendCacheHitCopyingActor.scala +++ /dev/null @@ -1,83 +0,0 @@ -package cromwell.backend.impl.bcs.callcaching - -import com.google.cloud.storage.contrib.nio.CloudStorageOptions -import common.util.TryUtil -import cromwell.backend.BackendInitializationData -import cromwell.backend.impl.bcs.BcsBackendInitializationData -import cromwell.backend.io.JobPaths -import cromwell.backend.standard.callcaching.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams} -import cromwell.core.CallOutputs -import cromwell.core.io.{IoCommand, IoTouchCommand} -import cromwell.core.path.Path -import cromwell.core.simpleton.{WomValueBuilder, WomValueSimpleton} -import cromwell.filesystems.oss.batch.OssBatchCommandBuilder -import wom.values.WomFile - -import scala.language.postfixOps -import scala.util.Try - -class BcsBackendCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) extends StandardCacheHitCopyingActor(standardParams) { - override protected val commandBuilder: OssBatchCommandBuilder.type = OssBatchCommandBuilder - private val cachingStrategy = BackendInitializationData - .as[BcsBackendInitializationData](standardParams.backendInitializationDataOption) - .bcsConfiguration.duplicationStrategy - - override def processSimpletons(womValueSimpletons: Seq[WomValueSimpleton], - sourceCallRootPath: Path, - ): Try[(CallOutputs, Set[IoCommand[_]])] = cachingStrategy match { - case CopyCachedOutputs => super.processSimpletons(womValueSimpletons, sourceCallRootPath) - case UseOriginalCachedOutputs => - val touchCommands: Seq[Try[IoTouchCommand]] = womValueSimpletons collect { - case WomValueSimpleton(_, wdlFile: WomFile) => getPath(wdlFile.value) flatMap OssBatchCommandBuilder.touchCommand - } - - TryUtil.sequence(touchCommands) map { - WomValueBuilder.toJobOutputs(jobDescriptor.taskCall.outputPorts, womValueSimpletons) -> _.toSet - } - } - - override def processDetritus(sourceJobDetritusFiles: Map[String, String] - ): Try[(Map[String, Path], Set[IoCommand[_]])] = cachingStrategy match { - case CopyCachedOutputs => super.processDetritus(sourceJobDetritusFiles) - case UseOriginalCachedOutputs => - // apply getPath on each detritus string file - val detritusAsPaths = detritusFileKeys(sourceJobDetritusFiles).toSeq map { key => - key -> getPath(sourceJobDetritusFiles(key)) - } toMap - - // Don't forget to re-add the CallRootPathKey that has been filtered out by detritusFileKeys - TryUtil.sequenceMap(detritusAsPaths, "Failed to make paths out of job detritus") flatMap { newDetritus => - Try { - // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap - (newDetritus + (JobPaths.CallRootPathKey -> destinationCallRootPath)) -> - newDetritus.values.map(OssBatchCommandBuilder.touchCommand(_).get).toSet - } - } - } - - override protected def additionalIoCommands(sourceCallRootPath: Path, - originalSimpletons: Seq[WomValueSimpleton], - newOutputs: CallOutputs, - originalDetritus: Map[String, String], - newDetritus: Map[String, Path]): Try[List[Set[IoCommand[_]]]] = Try { - cachingStrategy match { - case UseOriginalCachedOutputs => - val content = - s""" - |This directory does not contain any output files because this job matched an identical job that was previously run, thus it was a cache-hit. - |Cromwell is configured to not copy outputs during call caching. To change this, edit the filesystems.gcs.caching.duplication-strategy field in your backend configuration. - |The original outputs can be found at this location: ${sourceCallRootPath.pathAsString} - """.stripMargin - - // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap - List(Set( - OssBatchCommandBuilder.writeCommand( - path = jobPaths.forCallCacheCopyAttempts.callExecutionRoot / "call_caching_placeholder.txt", - content = content, - options = Seq(CloudStorageOptions.withMimeType("text/plain")), - ).get - )) - case CopyCachedOutputs => List.empty - } - } -} diff --git a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/callcaching/BcsCacheHitDuplicationStrategy.scala b/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/callcaching/BcsCacheHitDuplicationStrategy.scala deleted file mode 100644 index 55ba639d6fd..00000000000 --- a/supportedBackends/bcs/src/main/scala/cromwell/backend/impl/bcs/callcaching/BcsCacheHitDuplicationStrategy.scala +++ /dev/null @@ -1,6 +0,0 @@ -package cromwell.backend.impl.bcs.callcaching - -sealed trait BcsCacheHitDuplicationStrategy - -case object CopyCachedOutputs extends BcsCacheHitDuplicationStrategy -case object UseOriginalCachedOutputs extends BcsCacheHitDuplicationStrategy diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsClusterIdOrConfigurationSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsClusterIdOrConfigurationSpec.scala deleted file mode 100644 index b57fa60344b..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsClusterIdOrConfigurationSpec.scala +++ /dev/null @@ -1,106 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.prop.Tables.Table -import org.scalatest.prop.TableDrivenPropertyChecks._ -import org.scalatest.TryValues._ - -import scala.util.Failure - - -class BcsClusterIdOrConfigurationSpec extends BcsTestUtilSpec { - behavior of s"BcsClusterIdOrConfiguration" - - val clusterIdTable = Table( - ("unparsed", "parsed"), - ("cls-xxxx", Option("cls-xxxx")), - ("job-xxxx", None) - ) - - it should "parse correct cluster id" in { - forAll(clusterIdTable) { (unparsed, parsed) => - BcsClusterIdOrConfiguration.idPattern.findFirstIn(unparsed) shouldEqual(parsed) - } - } - - val resourceTypeTable = Table( - ("unparsed", "parsed"), - ("OnDemand", Option("OnDemand")), - ("Spot", Option("Spot")), - ("Other", None) - ) - - it should "parse correct resource type" in { - forAll(resourceTypeTable) { (unparsed, parsed) => - BcsClusterIdOrConfiguration.resourceTypePattern.findFirstIn(unparsed) shouldEqual(parsed) - } - } - - - val instanceTypeTable = Table( - ("unparsed", "parsed"), - ("ecs.s1.large", Option("ecs.s1.large")), - ("bcs.s1.large", Option("bcs.s1.large")) - ) - - it should "parse correct instance type" in { - forAll(instanceTypeTable) { (unparsed, parsed) => - BcsClusterIdOrConfiguration.instanceTypePattern.findFirstIn(unparsed) shouldEqual parsed - } - } - - val spotStrategyTable = Table( - ("unparsed", "parsed"), - ("SpotWithPriceLimit", Option("SpotWithPriceLimit")), - ("SpotAsPriceGo", Option("SpotAsPriceGo")) - ) - - - it should "parse correct spot strategy" in { - forAll(spotStrategyTable) { (unparsed, parsed) => - BcsClusterIdOrConfiguration.spotStrategyPattern.findFirstIn(unparsed) shouldEqual parsed - } - } - - val spotPriceLimitTable = Table( - ("unparsed", "parsed"), - ("1.0", Option(1.0.toFloat)), - ("0.1", Option(0.1.toFloat)), - ("0.12", Option(0.12.toFloat)), - ("0.123", Option(0.123.toFloat)) - ) - - it should "parse correct spot price limit" in { - forAll(spotPriceLimitTable) { (unparsed, parsed) => - BcsClusterIdOrConfiguration.spotPriceLimitPattern.findFirstIn(unparsed) map {limit => limit.toFloat} shouldEqual parsed - } - } - - val validClusterInfoTable = Table( - ("unparsed", "parsed"), - ("cls-id", Left("cls-id")), - ("OnDemand ecs.s1.large img-test", Right(AutoClusterConfiguration("OnDemand", "ecs.s1.large", "img-test"))), - ("OnDemand ecs.s1.large img-test cls-test", Right(AutoClusterConfiguration("OnDemand", "ecs.s1.large", "img-test", clusterId = Option("cls-test")))), - ("ecs.s1.large img-test", Right(AutoClusterConfiguration("OnDemand", "ecs.s1.large", "img-test"))), - ("ecs.s1.large img-test cls-test", Right(AutoClusterConfiguration("OnDemand", "ecs.s1.large", "img-test", clusterId = Option("cls-test")))), - ("Spot ecs.s1.large img-test SpotWithPriceLimit 0.1", Right(AutoClusterConfiguration("Spot", "ecs.s1.large", "img-test", Option("SpotWithPriceLimit"), Option(0.1.toFloat)))), - ("Spot ecs.s1.large img-test SpotWithPriceLimit 0.1 cls-test", Right(AutoClusterConfiguration("Spot", "ecs.s1.large", "img-test", Option("SpotWithPriceLimit"), Option(0.1.toFloat), Option("cls-test")))), - ("Spot ecs.s1.large img-test SpotAsPriceGo 0.1", Right(AutoClusterConfiguration("Spot", "ecs.s1.large", "img-test", Option("SpotAsPriceGo"), Option(0.1.toFloat)))), - ("Spot ecs.s1.large img-test SpotAsPriceGo 0.1 cls-test", Right(AutoClusterConfiguration("Spot", "ecs.s1.large", "img-test", Option("SpotAsPriceGo"), Option(0.1.toFloat), Option("cls-test")))), - - ) - - - it should "parse correct cluster id or cluster configuration" in { - forAll(validClusterInfoTable) { (unparsed, parsed) => - BcsClusterIdOrConfiguration.parse(unparsed).success.value shouldEqual parsed - } - } - - val invalidClusterInfos = List("OnDemand", "", "OnDemand other", "other ecs.s1.large") - - invalidClusterInfos foreach { unparsed => - it should s"throw when parsing $unparsed" in { - BcsClusterIdOrConfiguration.parse(unparsed) shouldBe a [Failure[_]] - } - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsConfigurationSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsConfigurationSpec.scala deleted file mode 100644 index ad32eec7cce..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsConfigurationSpec.scala +++ /dev/null @@ -1,48 +0,0 @@ -package cromwell.backend.impl.bcs - -import com.typesafe.config.ConfigValueFactory -import cromwell.backend.impl.bcs.callcaching.UseOriginalCachedOutputs - -class BcsConfigurationSpec extends BcsTestUtilSpec { - behavior of "BcsConfiguration" - type ValueOrDelete = Either[Boolean, AnyRef] - - def backendConfiguration = BcsTestUtilSpec.BcsBackendConfigurationDescriptor - def defaultBackendConfig = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.backendConfig - - it should "have correct bcs region" in { - val region = "cn-hangzhou" - val configs = Map("region" -> Right(region)) - val conf = withConfig(configs) - conf.bcsRegion shouldEqual Some(region) - } - - it should "have correct bcs access id and key" in { - val id = "test-access-id" - val key = "test-access-key" - val configs = Map("access-id" -> Right(id), "access-key" -> Right(key)) - val conf = withConfig(configs) - conf.bcsAccessId shouldEqual Some(id) - conf.bcsAccessKey shouldEqual Some(key) - } - - it should "have correct bcs callcaching strategy" in { - val region = "cn-hangzhou" - val configs = Map("region" -> Right(region)) - val conf = withConfig(configs) - conf.duplicationStrategy shouldEqual UseOriginalCachedOutputs - } - - - private def withConfig(configs: Map[String, ValueOrDelete]) = { - var descriptor = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.copy() - for ((key, value) <- configs) { - value match { - case Left(_) => descriptor = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.copy(backendConfig = descriptor.backendConfig.withoutPath(key)) - case Right(v) => descriptor = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.copy(backendConfig = descriptor.backendConfig.withValue(key, ConfigValueFactory.fromAnyRef(v))) - } - } - new BcsConfiguration(descriptor) - } - -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsDiskSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsDiskSpec.scala deleted file mode 100644 index 98f6b1b3449..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsDiskSpec.scala +++ /dev/null @@ -1,36 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.prop.Tables.Table -import org.scalatest.prop.TableDrivenPropertyChecks._ -import org.scalatest.TryValues._ - -import scala.util.Failure - -class BcsDiskSpec extends BcsTestUtilSpec { - behavior of s"BcsDisk" - - val validDiskTable = Table( - ("unparsed", "parsed"), - ("cloud 40", BcsSystemDisk("cloud", 40)), - ("cloud 200 /home/inputs/", BcsDataDisk("cloud", 200, "/home/inputs/")) - ) - - it should "parse correct disk" in { - forAll(validDiskTable) { (unparsed, parsed)=> - BcsDisk.parse(unparsed).success.value shouldEqual(parsed) - } - } - - val invalidDiskTable = List( - "", - "cloud", - "40", - "cloud 40GB", - "cloud /home/inputs/", - "cloud /home/inputs 40" - ) - - invalidDiskTable foreach { unparsed => - BcsDisk.parse(unparsed) shouldBe a [Failure[_]] - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsDockerSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsDockerSpec.scala deleted file mode 100644 index 1a3217111b5..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsDockerSpec.scala +++ /dev/null @@ -1,22 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.prop.Tables.Table -import org.scalatest.prop.TableDrivenPropertyChecks._ -import org.scalatest.TryValues._ - - -class BcsDockerSpec extends BcsTestUtilSpec { - behavior of s"BcsDocker" - - val validDockerTable = Table( - ("unparsed", "parsed"), - ("ubuntu/latest oss://bcs-reg/ubuntu/", BcsDockerWithPath("ubuntu/latest", "oss://bcs-reg/ubuntu/")), - ("ubuntu/latest", BcsDockerWithoutPath("ubuntu/latest")) - ) - - it should "parse correct docker configuration" in { - forAll(validDockerTable) { (unparsed, parsed) => - BcsDocker.parse(unparsed).success.value shouldEqual(parsed) - } - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsJobPathsSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsJobPathsSpec.scala deleted file mode 100644 index 15148f39a35..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsJobPathsSpec.scala +++ /dev/null @@ -1,25 +0,0 @@ -package cromwell.backend.impl.bcs - -import common.mock.MockSugar -import cromwell.filesystems.oss.OssPath -import org.mockito.Mockito._ - -class BcsJobPathsSpec extends BcsTestUtilSpec with MockSugar { - behavior of s"BcsJobPathsSpec" - - var root: OssPath = mockPathBuilder.build("oss://bcs-test/root/").getOrElse(throw new IllegalArgumentException()) - - var workflowPath: BcsWorkflowPaths = { - val workflowPaths = mock[BcsWorkflowPaths] - - when(workflowPaths.workflowRoot).thenReturn(root) - workflowPaths - } - - def name = "test" - - it should "have right package name" in { - val jobPath = BcsJobPaths(workflowPath, jobKey) - jobPath.workerPath shouldEqual jobPath.callRoot.resolve(s"${jobPath.workerFileName}") - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsJobSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsJobSpec.scala deleted file mode 100644 index ad4ec85b64f..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsJobSpec.scala +++ /dev/null @@ -1,137 +0,0 @@ -package cromwell.backend.impl.bcs - -import com.aliyuncs.batchcompute.main.v20151111.BatchComputeClient -import com.aliyuncs.batchcompute.pojo.v20151111.TaskDescription -import common.mock.MockSugar -import wom.values._ - - -class BcsJobSpec extends BcsTestUtilSpec with MockSugar { - - behavior of s"BcsJob" - - val mockBcsClient = mock[BatchComputeClient] - val name = "cromwell" - val description = name - val command = "python main.py" - val packagePath = mockPathBuilder.build("oss://bcs-test/worker.tar.gz").get - val mounts = Seq.empty[BcsMount] - val envs = Map.empty[String, String] - - it should "have correct name and other basic info" in { - val job = withRuntime() - job.jobDesc.getName shouldEqual name - job.jobDesc.getDescription shouldEqual description - job.jobDesc.getType shouldEqual "DAG" - - val task = taskWithRuntime() - task.getParameters.getCommand.getCommandLine shouldEqual command - task.getParameters.getCommand.getPackagePath shouldEqual packagePath.pathAsString - - } - - it should "have correct auto release option" in { - val runtime = Map("autoReleasejob" -> WomBoolean(false)) - withRuntime(runtime).jobDesc.isAutoRelease shouldBe false - } - - it should "have correct timeout" in { - val timeout = 3000 - val runtime = Map("timeout" -> WomInteger(timeout)) - taskWithRuntime(runtime).getTimeout shouldEqual timeout - } - - it should "have correct mounts" in { - val src = "oss://bcs-job/dir/" - val dest = "/home/inputs/" - val writeSupport = false - val runtime = Map("mounts" -> WomString(s"$src $dest $writeSupport")) - taskWithRuntime(runtime).getMounts().getEntries should have size(1) - taskWithRuntime(runtime).getMounts().getEntries.get(0).getSource shouldBe src - taskWithRuntime(runtime).getMounts().getEntries.get(0).getDestination shouldBe dest - taskWithRuntime(runtime).getMounts().getEntries.get(0).isWriteSupport shouldBe writeSupport - } - - it should "have correct cluster id" in { - val clusterId = "cls-bcs" - val runtime = Map("cluster" -> WomString(clusterId)) - taskWithRuntime(runtime).getClusterId shouldEqual clusterId - } - - it should "have correct docker option" in { - val dockerImage = "ubuntu/latest" - val dockerPath = "oss://bcs-reg/ubuntu/".toLowerCase() - val runtime = Map("dockerTag" -> WomString(s"$dockerImage $dockerPath")) - taskWithRuntime(runtime).getParameters.getCommand.getEnvVars.get(BcsJob.BcsDockerImageEnvKey) shouldEqual null - taskWithRuntime(runtime).getParameters.getCommand.getEnvVars.get(BcsJob.BcsDockerPathEnvKey) shouldEqual null - } - - it should "have correct auto cluster configuration" in { - val resourceType = "Spot" - val instanceType = "ecs.c1.large" - val imageId = "img-centos" - val spotStrategy = "SpotWithPriceLimit" - val spotPriceLimit = 0.12 - val cluster = s"$resourceType $instanceType $imageId $spotStrategy $spotPriceLimit" - val imageIdForCallCaching = "img-ubuntu-vpc" - val reserveOnFail = true - val cidr = "172.16.16.0/20" - val vpcId = "vpc-test" - val systemDiskType = "cloud" - val systemDiskSize = 40 - val dataDiskType = "cloud_efficiency" - val dataDiskSize = 250 - val dataDiskMountPoint = "/home/data/" - val userDataKey = "key" - val userDataValue = "value" - - val runtime = Map( - "cluster" -> WomString(cluster), - "reserveOnFail" -> WomBoolean(reserveOnFail), - "vpc" -> WomString(s"$cidr $vpcId"), - "systemDisk" -> WomString(s"$systemDiskType $systemDiskSize"), - "dataDisk" -> WomString(s"$dataDiskType $dataDiskSize $dataDiskMountPoint"), - "userData" -> WomString(s"$userDataKey $userDataValue"), - "imageId" -> WomString(s"$imageIdForCallCaching") - ) - - val task = taskWithRuntime(runtime) - a [NullPointerException] should be thrownBy task.getClusterId.isEmpty - - val autoCluster = task.getAutoCluster - autoCluster.isReserveOnFail shouldEqual reserveOnFail - autoCluster.getImageId shouldEqual imageIdForCallCaching - autoCluster.getResourceType shouldEqual resourceType - autoCluster.getInstanceType shouldEqual instanceType - autoCluster.getSpotStrategy shouldEqual spotStrategy - autoCluster.getSpotPriceLimit shouldEqual spotPriceLimit.toFloat - - val vpc = autoCluster.getConfigs.getNetworks.getVpc - vpc.getVpcId shouldEqual vpcId - vpc.getCidrBlock shouldEqual cidr - - val systemDisk = autoCluster.getConfigs.getDisks.getSystemDisk - systemDisk.getType shouldEqual systemDiskType - systemDisk.getSize shouldEqual systemDiskSize - - val dataDisk = autoCluster.getConfigs.getDisks.getDataDisk - dataDisk.getType shouldEqual dataDiskType - dataDisk.getSize shouldEqual dataDiskSize - dataDisk.getMountPoint shouldEqual dataDiskMountPoint - - val userData = autoCluster.getUserData - userData.get(userDataKey) shouldEqual userDataValue - } - - - private def withRuntime(runtime: Map[String, WomValue] = Map.empty[String, WomValue]): BcsJob = { - val runtimeAttributes = createBcsRuntimeAttributes(runtime) - BcsJob(name, description, command, packagePath, runtimeAttributes.mounts.getOrElse(mounts), envs, runtimeAttributes, None, None, mockBcsClient) - } - - private def taskWithRuntime(runtime: Map[String, WomValue] = Map.empty[String, WomValue]): TaskDescription = { - val job = withRuntime(runtime) - job.jobDesc.getDag.getTasks.get("cromwell") - } - -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsMountSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsMountSpec.scala deleted file mode 100644 index c27d6488965..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsMountSpec.scala +++ /dev/null @@ -1,60 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.TryValues._ - -class BcsMountSpec extends BcsTestUtilSpec { - behavior of s"BcsMountSpec" - val ossObject = "oss://bcs-test/bcs-dir/" - val localFile = "/home/admin/local-dir/" - - it should "be an input mount if src starts with oss://" in { - var writeSupport = true - var entryString = s"$ossObject $localFile $writeSupport" - var entry = BcsMount.parse(entryString).success.value - - - entry shouldBe a [BcsInputMount] - BcsMount.toString(entry.src) shouldEqual ossObject - BcsMount.toString(entry.dest) shouldEqual localFile - entry.writeSupport shouldEqual writeSupport - - writeSupport = false - - entryString = s"$ossObject $localFile $writeSupport" - entry = BcsMount.parse(entryString).success.value - entry shouldBe a [BcsInputMount] - BcsMount.toString(entry.src) shouldEqual ossObject - BcsMount.toString(entry.dest) shouldEqual localFile - entry.writeSupport shouldEqual writeSupport - } - - it should "be an output mount if dest starts with oss://" in { - var writeSupport = true - var entryString = s"$localFile $ossObject $writeSupport" - var entry = BcsMount.parse(entryString).success.value - - - entry shouldBe a [BcsOutputMount] - BcsMount.toString(entry.src) shouldEqual localFile - BcsMount.toString(entry.dest) shouldEqual ossObject - entry.writeSupport shouldEqual writeSupport - - writeSupport = false - - entryString = s"$localFile $ossObject $writeSupport" - entry = BcsMount.parse(entryString).success.value - entry shouldBe a [BcsOutputMount] - BcsMount.toString(entry.src) shouldEqual localFile - BcsMount.toString(entry.dest) shouldEqual ossObject - entry.writeSupport shouldEqual writeSupport - } - - it should "throw if src and dest are both local files or oss files" in { - val writeSupport = true - var entryString = s"$localFile $localFile $writeSupport" - a [UnsupportedOperationException] should be thrownBy BcsMount.parse(entryString).failure.get - - entryString = s"$ossObject $ossObject $writeSupport" - a [UnsupportedOperationException] should be thrownBy BcsMount.parse(entryString).failure.get - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsRuntimeAttributesSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsRuntimeAttributesSpec.scala deleted file mode 100644 index 012c6a15b3b..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsRuntimeAttributesSpec.scala +++ /dev/null @@ -1,148 +0,0 @@ -package cromwell.backend.impl.bcs - -import wom.values._ - -class BcsRuntimeAttributesSpec extends BcsTestUtilSpec { - behavior of "BcsRuntimeAttributes" - - it should "build correct default runtime attributes from config string" in { - val runtime = Map.empty[String, WomValue] - val defaults = createBcsRuntimeAttributes(runtime) - defaults shouldEqual expectedRuntimeAttributes - } - - it should "parse dockerTag without docker path" in { - val runtime = Map("dockerTag" -> WomString("ubuntu/latest")) - val expected = expectedRuntimeAttributes.copy(dockerTag = Option(BcsDockerWithoutPath("ubuntu/latest"))) - createBcsRuntimeAttributes(runtime) shouldEqual(expected) - } - - it should "parse dockerTag with path" in { - val runtime = Map("dockerTag" -> WomString("centos/latest oss://bcs-dir/registry/")) - val expected = expectedRuntimeAttributes.copy(dockerTag = Option(BcsDockerWithPath("centos/latest", "oss://bcs-dir/registry/"))) - createBcsRuntimeAttributes(runtime) shouldEqual(expected) - } - - it should "parse dockerTag fail if an empty string value" in { - val runtime = Map("dockerTag" -> WomString("")) - an [Exception] should be thrownBy createBcsRuntimeAttributes(runtime) - } - - it should "parse docker" in { - val runtime = Map("docker" -> WomString("registry.cn-beijing.aliyuncs.com/test/testubuntu:0.2")) - val expected = expectedRuntimeAttributes.copy(docker = Option(BcsDockerWithoutPath("registry.cn-beijing.aliyuncs.com/test/testubuntu:0.2"))) - createBcsRuntimeAttributes(runtime) shouldEqual(expected) - } - - it should "parse correct user data" in { - val runtime = Map("userData" -> WomString("key value1")) - val expected = expectedRuntimeAttributes.copy(userData = Option(Vector(BcsUserData("key", "value1")))) - createBcsRuntimeAttributes(runtime) shouldEqual(expected) - } - - it should "throw if user data is invalid" in { - val runtime = Map("userData" -> WomString("keyvalue")) - an [Exception] should be thrownBy createBcsRuntimeAttributes(runtime) - } - - it should "parse correct input mount" in { - val runtime = Map("mounts" -> WomString("oss://bcs-dir/bcs-file /home/inputs/input_file false")) - val expected = expectedRuntimeAttributes.copy(mounts = Option(Vector(BcsInputMount(Left(mockPathBuilder.build("oss://bcs-dir/bcs-file").get), Right("/home/inputs/input_file"), false)))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct out mount" in { - val runtime = Map("mounts" -> WomString("/home/outputs/ oss://bcs-dir/outputs/ true")) - val expected = expectedRuntimeAttributes.copy(mounts = Option(Vector(BcsOutputMount(Right("/home/outputs/"), Left(mockPathBuilder.build("oss://bcs-dir/outputs/").get), true)))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "throw if mounts is invalid" in { - val runtime = Map("mounts" -> WomString("invalid mounts")) - an [Exception] should be thrownBy createBcsRuntimeAttributes(runtime) - } - - it should "parse correct cluster id" in { - val runtime = Map("cluster" -> WomString("cls-1")) - val expected = expectedRuntimeAttributes.copy(cluster = Option(Left("cls-1"))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct ondemand auto cluster configuration" in { - val runtime = Map("cluster" -> WomString("OnDemand ecs.s1.large img-ubuntu")) - val expected = expectedRuntimeAttributes.copy(cluster = Option(Right(AutoClusterConfiguration("OnDemand", "ecs.s1.large", "img-ubuntu")))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct spot auto cluster configuration" in { - val runtime = Map("cluster" -> WomString("Spot ecs.s1.large img-ubuntu")) - val expected = expectedRuntimeAttributes.copy(cluster = Option(Right(AutoClusterConfiguration("Spot", "ecs.s1.large", "img-ubuntu")))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - - } - - it should "parse correct spot auto cluster price option" in { - val runtime = Map("cluster" -> WomString("Spot ecs.s1.large img-ubuntu SpotWithPriceLimit 0.1")) - val expected = expectedRuntimeAttributes.copy(cluster = Option(Right(AutoClusterConfiguration("Spot", "ecs.s1.large", "img-ubuntu", Option("SpotWithPriceLimit"), Some(0.1.toFloat))))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct vpc cidr block" in { - val runtime = Map("vpc" -> WomString("172.16.16.0/20")) - val expected = expectedRuntimeAttributes.copy(vpc = Option(BcsVpcConfiguration(Option("172.16.16.0/20")))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct vpc id" in { - val runtime = Map("vpc" -> WomString("vpc-xxxx")) - val expected = expectedRuntimeAttributes.copy(vpc = Option(BcsVpcConfiguration(vpcId = Option("vpc-xxxx")))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct system disk" in { - val runtime = Map("systemDisk" -> WomString("cloud_efficiency 250")) - val expected = expectedRuntimeAttributes.copy(systemDisk = Option(BcsSystemDisk("cloud_efficiency", 250))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "throw when parsing invalid system disk" in { - val runtime = Map("systemDisk" -> WomString("cloud_efficiency 250 /home/data/")) - an [Exception] should be thrownBy createBcsRuntimeAttributes(runtime) - } - - it should "parse correct data disk" in { - val runtime = Map("dataDisk" -> WomString("cloud 400 /home/data/")) - val expected = expectedRuntimeAttributes.copy(dataDisk = Option(BcsDataDisk("cloud", 400, "/home/data/"))) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "throw when parsing invalid data disk" in { - val runtime = Map("dataDisk" -> WomString("cloud_efficiency 250")) - an [Exception] should be thrownBy createBcsRuntimeAttributes(runtime) - } - - it should "parse correct reserve on fail option" in { - val runtime = Map("reserveOnFail" -> WomBoolean(false)) - val expected = expectedRuntimeAttributes.copy(reserveOnFail = Option(false)) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct auto release option" in { - val runtime = Map("autoReleaseJob" -> WomBoolean(false)) - val expected = expectedRuntimeAttributes.copy(autoReleaseJob = Option(false)) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct verbose option" in { - val runtime = Map("verbose" -> WomBoolean(false)) - val expected = expectedRuntimeAttributes.copy(verbose = Option(false)) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - - it should "parse correct time out" in { - val runtime = Map("timeout" -> WomInteger(3000)) - val expected = expectedRuntimeAttributes.copy(timeout = Option(3000)) - createBcsRuntimeAttributes(runtime) shouldEqual expected - } - -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsTestUtilSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsTestUtilSpec.scala deleted file mode 100644 index 61da2f6aba0..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsTestUtilSpec.scala +++ /dev/null @@ -1,183 +0,0 @@ -package cromwell.backend.impl.bcs - -import com.typesafe.config.{Config, ConfigFactory} -import common.collections.EnhancedCollections._ -import cromwell.backend.BackendSpec.buildWdlWorkflowDescriptor -import cromwell.backend.validation.ContinueOnReturnCodeSet -import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, RuntimeAttributeDefinition} -import cromwell.core.{TestKitSuite, WorkflowOptions} -import cromwell.filesystems.oss.OssPathBuilder -import cromwell.filesystems.oss.nio.DefaultOssStorageConfiguration -import cromwell.util.SampleWdl -import org.scalatest.BeforeAndAfter -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers -import org.slf4j.helpers.NOPLogger -import spray.json.{JsObject, JsString} -import wom.values.WomValue - -object BcsTestUtilSpec { - - val DefaultRunAttributesString: String = - """ - |default-runtime-attributes { - | failOnStderr: false - | continueOnReturnCode: 0 - | cluster: "cls-mycluster" - | mounts: "oss://bcs-bucket/bcs-dir/ /home/inputs/ false" - | dockerTag: "ubuntu/latest oss://bcs-reg/ubuntu/" - | docker: "registry.cn-beijing.aliyuncs.com/test/testubuntu:0.1" - | userData: "key value" - | reserveOnFail: true - | autoReleaseJob: true - | verbose: false - | systemDisk: "cloud 50" - | dataDisk: "cloud 250 /home/data/" - | timeout: 3000 - | vpc: "192.168.0.0/16 vpc-xxxx" - | tag: "jobTag" - | imageId: "img-ubuntu-vpc" - | isv: "test-isv" - |} - """.stripMargin - - val BcsBackendConfigString: String = - s""" - |root = "oss://your-bucket/cromwell-exe" - |dockerRoot = "/cromwell-executions" - |region = "" - | - |access-id = "" - |access-key = "" - |security-token = "" - | - |filesystems { - | oss { - | auth { - | endpoint = "" - | access-id = "" - | access-key = "" - | security-token = "" - | } - | caching { - | duplication-strategy = "reference" - | } - | } - |} - | - |$DefaultRunAttributesString - | - |""".stripMargin - - val BcsBackendConfigWithoutDefaultString: String = - s""" - |root = "oss://your-bucket/cromwell-exe" - |dockerRoot = "/cromwell-executions" - |region = "" - | - |access-id = "" - |access-key = "" - |security-token = "" - | - |filesystems { - | oss { - | auth { - | endpoint = "" - | access-id = "" - | access-key = "" - | security-token = "" - | } - | } - |} - | - |""".stripMargin - - val BcsGlobalConfigString: String = - s""" - |backend { - | default = "BCS" - | providers { - | BCS { - | actor-factory = "cromwell.backend.impl.bcs.BcsBackendLifecycleActorFactory" - | config { - | $BcsBackendConfigString - | } - | } - | } - |} - | - |""".stripMargin - - val BcsBackendConfig: Config = ConfigFactory.parseString(BcsBackendConfigString) - val BcsGlobalConfig: Config = ConfigFactory.parseString(BcsGlobalConfigString) - val BcsBackendConfigWithoutDefault: Config = ConfigFactory.parseString(BcsBackendConfigWithoutDefaultString) - val BcsBackendConfigurationDescriptor: BackendConfigurationDescriptor = - BackendConfigurationDescriptor(BcsBackendConfig, BcsGlobalConfig) - val BcsBackendConfigurationWithoutDefaultDescriptor: BackendConfigurationDescriptor = - BackendConfigurationDescriptor(BcsBackendConfigWithoutDefault, BcsGlobalConfig) - val EmptyWorkflowOption: WorkflowOptions = WorkflowOptions.fromMap(Map.empty).get -} - -trait BcsTestUtilSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with BeforeAndAfter { - - before { - BcsMount.pathBuilders = List(mockPathBuilder) - } - - val jobId = "test-bcs-job" - val mockOssConf: DefaultOssStorageConfiguration = - DefaultOssStorageConfiguration("oss.aliyuncs.com", "test-id", "test-key") - lazy val mockPathBuilder: OssPathBuilder = OssPathBuilder(mockOssConf) - val mockPathBuilders = List(mockPathBuilder) - lazy val workflowDescriptor: BackendWorkflowDescriptor = buildWdlWorkflowDescriptor( - SampleWdl.HelloWorld.workflowSource(), - inputFileAsJson = Option(JsObject(SampleWdl.HelloWorld.rawInputs.safeMapValues(JsString.apply)).compactPrint) - ) - lazy val jobKey: BackendJobDescriptorKey = { - val call = workflowDescriptor.callable.taskCallNodes.head - BackendJobDescriptorKey(call, None, 1) - } - - - val expectedContinueOnReturn: ContinueOnReturnCodeSet = ContinueOnReturnCodeSet(Set(0)) - val expectedDockerTag: Option[BcsDockerWithPath] = - Option(BcsDockerWithPath("ubuntu/latest", "oss://bcs-reg/ubuntu/")) - val expectedDocker: Option[BcsDockerWithoutPath] = - Option(BcsDockerWithoutPath("registry.cn-beijing.aliyuncs.com/test/testubuntu:0.1")) - val expectedFailOnStderr = false - val expectedUserData: Option[Vector[BcsUserData]] = Option(Vector(new BcsUserData("key", "value"))) - val expectedMounts: Option[Vector[BcsInputMount]] = - Option(Vector( - BcsInputMount( - src = Left(mockPathBuilder.build("oss://bcs-bucket/bcs-dir/").get), - dest = Right("/home/inputs/"), - writeSupport = false, - ) - )) - val expectedCluster: Option[Left[String, Nothing]] = Option(Left("cls-mycluster")) - val expectedImageId: Option[String] = Option("img-ubuntu-vpc") - val expectedSystemDisk: Option[BcsSystemDisk] = Option(BcsSystemDisk("cloud", 50)) - val expectedDataDisk: Option[BcsDataDisk] = Option(BcsDataDisk("cloud", 250, "/home/data/")) - - val expectedReserveOnFail: Option[Boolean] = Option(true) - val expectedAutoRelease: Option[Boolean] = Option(true) - val expectedTimeout: Option[Int] = Option(3000) - val expectedVerbose: Option[Boolean] = Option(false) - val expectedVpc: Option[BcsVpcConfiguration] = - Option(BcsVpcConfiguration(Option("192.168.0.0/16"), Option("vpc-xxxx"))) - val expectedTag: Option[String] = Option("jobTag") - val expectedIsv: Option[String] = Option("test-isv") - - - val expectedRuntimeAttributes = new BcsRuntimeAttributes(expectedContinueOnReturn, expectedDockerTag, expectedDocker, expectedFailOnStderr, expectedMounts, expectedUserData, expectedCluster, - expectedImageId, expectedSystemDisk, expectedDataDisk, expectedReserveOnFail, expectedAutoRelease, expectedTimeout, expectedVerbose, expectedVpc, expectedTag, expectedIsv) - - - protected def createBcsRuntimeAttributes(runtimeAttributes: Map[String, WomValue]): BcsRuntimeAttributes = { - val builder = BcsRuntimeAttributes.runtimeAttributesBuilder(BcsTestUtilSpec.BcsBackendConfigurationDescriptor.backendRuntimeAttributesConfig) - val default = RuntimeAttributeDefinition.addDefaultsToAttributes( - builder.definitions.toSet, BcsTestUtilSpec.EmptyWorkflowOption)(runtimeAttributes) - val validated = builder.build(default, NOPLogger.NOP_LOGGER) - BcsRuntimeAttributes(validated, BcsTestUtilSpec.BcsBackendConfigurationDescriptor.backendRuntimeAttributesConfig) - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsUserDataSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsUserDataSpec.scala deleted file mode 100644 index 6392f60dd35..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsUserDataSpec.scala +++ /dev/null @@ -1,18 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.TryValues._ - -class BcsUserDataSpec extends BcsTestUtilSpec { - behavior of s"BcsUserDataSpec" - - it should "work for right user data" in { - val key = "key" - val value = "value" - - val userData = BcsUserData.parse(s"$key $value") - userData.success.value.key shouldEqual key - userData.success.value.value shouldEqual value - - BcsUserData.parse(s"$key$value").isFailure shouldBe true - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsVpcConfigurationSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsVpcConfigurationSpec.scala deleted file mode 100644 index 94bc48ecbf2..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsVpcConfigurationSpec.scala +++ /dev/null @@ -1,23 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.prop.Tables.Table -import org.scalatest.prop.TableDrivenPropertyChecks._ -import org.scalatest.TryValues._ - -class BcsVpcConfigurationSpec extends BcsTestUtilSpec { - behavior of s"BcsVpcConfiguration" - - val validVpcTable = Table( - ("unparsed", "parsed"), - ("192.168.0.0/16", BcsVpcConfiguration(Some("192.168.0.0/16"), None)), - ("vpc-xxxx", BcsVpcConfiguration(None, Some("vpc-xxxx"))), - ("192.168.0.0/16 vpc-xxxx", BcsVpcConfiguration(Some("192.168.0.0/16"), Some("vpc-xxxx"))), - ("vpc-xxxx 192.168.0.0/16 ", BcsVpcConfiguration(Some("192.168.0.0/16"), Some("vpc-xxxx"))), - ) - - it should "parse correct vpc configuration" in { - forAll(validVpcTable) { (unparsed, parsed) => - BcsVpcConfiguration.parse(unparsed).success.value shouldEqual parsed - } - } -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsWorkflowPathsSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsWorkflowPathsSpec.scala deleted file mode 100644 index 4222055ab1a..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/BcsWorkflowPathsSpec.scala +++ /dev/null @@ -1,31 +0,0 @@ -package cromwell.backend.impl.bcs - -class BcsWorkflowPathsSpec extends BcsTestUtilSpec { - - behavior of s"BcsWorkflowPaths" - - it should "have correct input workflow mapping" in { - - import BcsTestUtilSpec._ - - val paths = BcsWorkflowPaths(workflowDescriptor, BcsBackendConfig, mockPathBuilders) - - val workflowInput = paths.getWorkflowInputMounts - workflowInput shouldBe a[BcsInputMount] - workflowInput.src shouldEqual(Left(paths.workflowRoot)) - BcsMount.toString(workflowInput.dest).startsWith(BcsJobPaths.BcsTempInputDirectory.pathAsString) shouldBe true - // DefaultPathBuilder always remove ending '/' from directory path. - BcsMount.toString(workflowInput.dest).endsWith(paths.workflowRoot.pathWithoutScheme.stripSuffix("/")) shouldBe true - } - - it should "have correct job paths" in { - - import BcsTestUtilSpec._ - - val paths = BcsWorkflowPaths(workflowDescriptor, BcsBackendConfig, mockPathBuilders) - - val jobPaths = paths.toJobPaths(paths, jobKey) - jobPaths shouldBe a [BcsJobPaths] - } - -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/RunStatusFactorySpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/RunStatusFactorySpec.scala deleted file mode 100644 index bb19a965971..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/RunStatusFactorySpec.scala +++ /dev/null @@ -1,34 +0,0 @@ -package cromwell.backend.impl.bcs - -import org.scalatest.TryValues._ - - - -final class RunStatusFactorySpec extends BcsTestUtilSpec { - behavior of s"RunStatusFactorySpec" - - private case class Status(str: String, - isRunningOrComplete: Boolean, - terminated: Boolean) - - strToClasses foreach { status => - it should behave like verifyStatus(status) - } - - def verifyStatus(status: Status) = { - it should s"have correct status: ${status.str}" in withClue(status.str) { - val s = RunStatusFactory.getStatus(jobId, status.str).success.value - s.status shouldEqual status.str - s.isRunningOrComplete shouldEqual status.isRunningOrComplete - s.isTerminated shouldEqual status.terminated - } - } - - private def strToClasses = Seq( - Status("Waiting", false, false), - Status("Running", true, false), - Status("Finished", true, true), - Status("Stopped", true, true), - Status("Failed", true, true) - ) -} diff --git a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/callcaching/BcsBackendCacheHitCopyingActorSpec.scala b/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/callcaching/BcsBackendCacheHitCopyingActorSpec.scala deleted file mode 100644 index dc06332aa22..00000000000 --- a/supportedBackends/bcs/src/test/scala/cromwell/backend/impl/bcs/callcaching/BcsBackendCacheHitCopyingActorSpec.scala +++ /dev/null @@ -1,83 +0,0 @@ -package cromwell.backend.impl.bcs.callcaching - - -import akka.actor.Props -import akka.testkit.TestActorRef -import com.typesafe.config.ConfigValueFactory -import common.mock.MockSugar -import cromwell.backend.impl.bcs.{BcsBackendInitializationData, BcsConfiguration, BcsRuntimeAttributes, BcsTestUtilSpec, BcsWorkflowPaths} -import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActorParams -import cromwell.core.path.Path -import wom.values._ -import cromwell.backend.impl.bcs.BcsTestUtilSpec.BcsBackendConfig -import cromwell.backend.standard.callcaching.DefaultStandardCacheHitCopyingActorParams -import cromwell.core.simpleton.WomValueSimpleton -import cromwell.filesystems.oss.OssPath -import org.mockito.Mockito._ - -import scala.util.Try - - -class BcsBackendCacheHitCopyingActorSpec extends BcsTestUtilSpec with MockSugar { - behavior of "BcsBackendCacheHitCopyingActor" - type ValueOrDelete = Either[Boolean, AnyRef] - - private val workflowPaths = BcsWorkflowPaths(workflowDescriptor, BcsBackendConfig, mockPathBuilders) - - private def buildInitializationData(configuration: BcsConfiguration) = { - - val runtimeAttributesBuilder = BcsRuntimeAttributes.runtimeAttributesBuilder(BcsTestUtilSpec.BcsBackendConfigurationDescriptor.backendRuntimeAttributesConfig) - BcsBackendInitializationData(workflowPaths, runtimeAttributesBuilder, configuration, null) - } - - private def withConfig(configs: Map[String, ValueOrDelete]) = { - var descriptor = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.copy() - for ((key, value) <- configs) { - value match { - case Left(_) => descriptor = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.copy(backendConfig = descriptor.backendConfig.withoutPath(key)) - case Right(v) => descriptor = BcsTestUtilSpec.BcsBackendConfigurationDescriptor.copy(backendConfig = descriptor.backendConfig.withValue(key, ConfigValueFactory.fromAnyRef(v))) - } - } - new BcsConfiguration(descriptor) - } - - - private val cacheHitCopyingActorParams = { - val mockCacheHitCopyingActorParams = mock[DefaultStandardCacheHitCopyingActorParams] - val id = "test-access-id" - val key = "test-access-key" - val configs = Map("access-id" -> Right(id), "access-key" -> Right(key)) - val conf = withConfig(configs) - when(mockCacheHitCopyingActorParams.backendInitializationDataOption).thenReturn(Option(buildInitializationData(conf))) - mockCacheHitCopyingActorParams - } - - class TestableBcsCacheHitCopyingActor(params: StandardCacheHitCopyingActorParams) - extends BcsBackendCacheHitCopyingActor(params) { - - val id = "test-access-id" - val key = "test-access-key" - val configs = Map("access-id" -> Right(id), "access-key" -> Right(key)) - - def this() = { - this(cacheHitCopyingActorParams) - } - - override def getPath(str: String): Try[Path] = mockPathBuilder.build("oss://bcs-dir/outputs/") - override def destinationJobDetritusPaths: Map[String, Path] = Map("stdout" - -> mockPathBuilder.build("oss://my-bucket/cromwell_dir/wf_echo/14e5dcd2-0c94-4035-aa7b-b90d7008202c/call-echo/stdout.log").get) - } - - it should "process simpleton and detritus correctly" in { - val simpleton = WomValueSimpleton("txt_files", WomSingleFile("oss://my-bucket/cromwell_dir/wf_echo/14e5dcd2-0c94-4035-aa7b-b90d7008202c/call-echo/abc.log")) - val detritus = Map("stdout" -> "oss://my-bucket/cromwell_dir/wf_echo/14e5dcd2-0c94-4035-aa7b-b90d7008202c/call-echo/stdout.log") - val sourceCallRootPath: OssPath = mockPathBuilder.build("oss://bcs-test/root/abc.log").getOrElse(throw new IllegalArgumentException()) - - val props = Props(new TestableBcsCacheHitCopyingActor()) - val cacheHitActor = TestActorRef[TestableBcsCacheHitCopyingActor]( - props, "TestableBcsCacheHitCopyingActor") - - noException should be thrownBy cacheHitActor.underlyingActor.processSimpletons(List(simpleton), sourceCallRootPath) - noException should be thrownBy cacheHitActor.underlyingActor.processDetritus(detritus) - } -} From b432e640e6978030c8a111119a267bdfc564c36c Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 5 Jul 2022 14:08:26 -0400 Subject: [PATCH 16/17] BW-1312 Fix incorrect documentation (#6790) --- docs/Imports.md | 4 ++-- src/ci/bin/test.inc.sh | 18 ++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/docs/Imports.md b/docs/Imports.md index 8cf8d930123..f929ada5883 100644 --- a/docs/Imports.md +++ b/docs/Imports.md @@ -10,7 +10,7 @@ There are two types of resources that are supported in imports: *http(s)* and *f ```wdl import "http://mywdlrepository/my.wdl" as http_import1 -import "https://github.com/broadinstitute/cromwell/blob/master/engine/src/main/resources/3step.wdl" as http_import2 +import "https://raw.githubusercontent.com/broadinstitute/cromwell/master/engine/src/main/resources/3step.wdl" as http_import2 ``` To use a file-based import resource, provide a ZIP bundle of your resources and then use a path relative to that ZIP in your import statement. For example: @@ -30,7 +30,7 @@ Here's a complete example showing both http(s) and file-based imports workflow i _workflow.wdl_ ```wdl -import "https://github.com/broadinstitute/cromwell/blob/master/engine/src/main/resources/3step.wdl" as http_import +import "https://raw.githubusercontent.com/broadinstitute/cromwell/master/engine/src/main/resources/3step.wdl" as http_import import "imports/imported.wdl" as provided_import workflow my_workflow { diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index eb52ab58ad1..95820513dfe 100644 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -158,8 +158,10 @@ cromwell::private::create_build_variables() { # will be the name of the branch targeted by the pull request, and for push builds it will be the name of the # branch. So, in case of push builds `git diff` will always return empty result. This is why we only use this short # circuiting logic for pull request builds - cromwell::private::set_variable_if_only_some_files_changed "^mkdocs.yml|^docs/|^CHANGELOG.md" "CROMWELL_BUILD_ONLY_DOCS_CHANGED" - cromwell::private::set_variable_if_only_some_files_changed "^src/ci/bin/testMetadataComparisonPython.sh|^scripts/" "CROMWELL_BUILD_ONLY_SCRIPTS_CHANGED" + + # PR #6790 disabled the conditional that skips tests for documentation-only PRs, because + # those PRs (and only those PRs) were uniformly failing tests with a nondescript error. + # https://broadinstitute.slack.com/archives/GHYJZ2ZE0/p1656625952888149?thread_ts=1656620572.975059&cid=GHYJZ2ZE0 case "${CROMWELL_BUILD_PROVIDER}" in "${CROMWELL_BUILD_PROVIDER_TRAVIS}") @@ -208,15 +210,9 @@ cromwell::private::create_build_variables() { # This allows quick sanity checks before starting PRs *and* publishing after merges into develop. if [[ "${travis_force_tests}" == "true" ]]; then CROMWELL_BUILD_RUN_TESTS=true - elif [[ "${CROMWELL_BUILD_ONLY_DOCS_CHANGED}" == "true" ]] && \ - [[ "${BUILD_TYPE}" != "checkPublish" ]]; then - CROMWELL_BUILD_RUN_TESTS=false elif [[ "${travis_minimal_tests}" == "true" ]] && \ [[ "${TRAVIS_EVENT_TYPE}" != "push" ]]; then CROMWELL_BUILD_RUN_TESTS=false - elif [[ "${CROMWELL_BUILD_ONLY_SCRIPTS_CHANGED}" == "true" ]] && \ - [[ "${BUILD_TYPE}" != "metadataComparisonPython" ]]; then - CROMWELL_BUILD_RUN_TESTS=false elif [[ "${TRAVIS_EVENT_TYPE}" == "push" ]] && \ [[ "${BUILD_TYPE}" != "sbt" ]]; then CROMWELL_BUILD_RUN_TESTS=false @@ -294,15 +290,9 @@ cromwell::private::create_build_variables() { # This allows quick sanity checks before starting PRs *and* publishing after merges into develop. if [[ "${circle_force_tests}" == "true" ]]; then CROMWELL_BUILD_RUN_TESTS=true - elif [[ "${CROMWELL_BUILD_ONLY_DOCS_CHANGED}" == "true" ]] && \ - [[ "${BUILD_TYPE}" != "checkPublish" ]]; then - CROMWELL_BUILD_RUN_TESTS=false elif [[ "${circle_minimal_tests}" == "true" ]] && \ [[ "${CROMWELL_BUILD_EVENT}" != "push" ]]; then CROMWELL_BUILD_RUN_TESTS=false - elif [[ "${CROMWELL_BUILD_ONLY_SCRIPTS_CHANGED}" == "true" ]] && \ - [[ "${BUILD_TYPE}" != "metadataComparisonPython" ]]; then - CROMWELL_BUILD_RUN_TESTS=false elif [[ "${CROMWELL_BUILD_EVENT}" == "push" ]] && \ [[ "${BUILD_TYPE}" != "sbt" ]]; then CROMWELL_BUILD_RUN_TESTS=false From 3e9aec2922f8a4b039dc729a9fd454178e4fb3f8 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 5 Jul 2022 20:10:48 -0400 Subject: [PATCH 17/17] BW-1228 Upgrade `protobuf-java` to non-vulnerable version (#6793) --- project/Dependencies.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0dbf9b4325f..3664f533ff7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -79,6 +79,10 @@ object Dependencies { private val mockitoV = "3.11.2" private val mockserverNettyV = "5.11.2" private val mouseV = "1.0.10" + /* + Newer version 8.0.29 fails `Control characters should work with metadata` Centaur tests, has charset changes mentioned in release notes + https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-29.html#mysqld-8-0-29-charset + */ private val mysqlV = "8.0.28" private val nettyV = "4.1.72.Final" private val owlApiV = "5.1.19" @@ -689,6 +693,10 @@ object Dependencies { "org.bouncycastle" % "bcprov-jdk15on" % "1.70", ) + private val protobufJavaOverrides = List( + "com.google.protobuf" % "protobuf-java" % "3.21.2", + ) + /* If we use a version in one of our projects, that's the one we want all the libraries to use ...plus other groups of transitive dependencies shared across multiple projects @@ -702,5 +710,6 @@ object Dependencies { scalaCollectionCompatOverrides ++ asyncHttpClientOverrides ++ nimbusdsOverrides ++ - bouncyCastleOverrides + bouncyCastleOverrides ++ + protobufJavaOverrides }