diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 0000000000..ad5b8c9554 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,34 @@ +pull_request_rules: + - name: Automatically merge on CI success and review + conditions: + - status-success=continuous-integration/travis-ci/pr + - "label=ready to merge" + - "approved-reviews-by=@oss-approvers" + actions: + merge: + method: squash + strict: smart + label: + add: ["auto merged"] + - name: Automatically merge PRs from maintainers on CI success and review + conditions: + - status-success=continuous-integration/travis-ci/pr + - "label=ready to merge" + - "author=@oss-approvers" + actions: + merge: + method: squash + strict: smart + label: + add: ["auto merged"] + - name: Automatically merge kork autobump PRs on CI success + conditions: + - status-success=continuous-integration/travis-ci/pr + - "label~=autobump-*" + - "author:spinnakerbot" + actions: + merge: + method: squash + strict: smart + label: + add: ["auto merged"] diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index d1e6146526..ba3ab40e20 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -9,6 +9,7 @@ ENV AWS_CLI_VERSION=1.16.208 RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y \ + openjdk-8-jre-headless \ curl \ python-pip && \ pip install awscli==${AWS_CLI_VERSION} --upgrade @@ -24,6 +25,6 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECT RUN curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/${KUBECTL_RELEASE}/${AWS_BINARY_RELEASE_DATE}/bin/linux/amd64/aws-iam-authenticator && \ chmod +x /usr/local/bin/aws-iam-authenticator -RUN adduser --disabled-login --system spinnaker +RUN adduser --disabled-login --system --group --uid 1000 spinnaker USER spinnaker CMD "/opt/halyard/bin/halyard" diff --git a/docs/commands.md b/docs/commands.md index 3990f4cb88..14808672f0 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -452,6 +452,23 @@ * [**hal config provider google disable**](#hal-config-provider-google-disable) * [**hal config provider google edit**](#hal-config-provider-google-edit) * [**hal config provider google enable**](#hal-config-provider-google-enable) + * [**hal config provider huaweicloud**](#hal-config-provider-huaweicloud) + * [**hal config provider huaweicloud account**](#hal-config-provider-huaweicloud-account) + * [**hal config provider huaweicloud account add**](#hal-config-provider-huaweicloud-account-add) + * [**hal config provider huaweicloud account delete**](#hal-config-provider-huaweicloud-account-delete) + * [**hal config provider huaweicloud account edit**](#hal-config-provider-huaweicloud-account-edit) + * [**hal config provider huaweicloud account get**](#hal-config-provider-huaweicloud-account-get) + * [**hal config provider huaweicloud account list**](#hal-config-provider-huaweicloud-account-list) + * [**hal config provider huaweicloud bakery**](#hal-config-provider-huaweicloud-bakery) + * [**hal config provider huaweicloud bakery base-image**](#hal-config-provider-huaweicloud-bakery-base-image) + * [**hal config provider huaweicloud bakery base-image add**](#hal-config-provider-huaweicloud-bakery-base-image-add) + * [**hal config provider huaweicloud bakery base-image delete**](#hal-config-provider-huaweicloud-bakery-base-image-delete) + * [**hal config provider huaweicloud bakery base-image edit**](#hal-config-provider-huaweicloud-bakery-base-image-edit) + * [**hal config provider huaweicloud bakery base-image get**](#hal-config-provider-huaweicloud-bakery-base-image-get) + * [**hal config provider huaweicloud bakery base-image list**](#hal-config-provider-huaweicloud-bakery-base-image-list) + * [**hal config provider huaweicloud bakery edit**](#hal-config-provider-huaweicloud-bakery-edit) + * [**hal config provider huaweicloud disable**](#hal-config-provider-huaweicloud-disable) + * [**hal config provider huaweicloud enable**](#hal-config-provider-huaweicloud-enable) * [**hal config provider kubernetes**](#hal-config-provider-kubernetes) * [**hal config provider kubernetes account**](#hal-config-provider-kubernetes-account) * [**hal config provider kubernetes account add**](#hal-config-provider-kubernetes-account-add) @@ -6886,6 +6903,7 @@ hal config provider [subcommands] * `docker-registry`: Manage and view Spinnaker configuration for the dockerRegistry provider * `ecs`: Manage and view Spinnaker configuration for the ecs provider * `google`: Manage and view Spinnaker configuration for the google provider + * `huaweicloud`: Manage and view Spinnaker configuration for the huaweicloud provider * `kubernetes`: Manage and view Spinnaker configuration for the kubernetes provider * `oracle`: Manage and view Spinnaker configuration for the oracle provider @@ -8918,6 +8936,354 @@ hal config provider google enable [parameters] * `--no-validate`: (*Default*: `false`) Skip validation. +--- +## hal config provider huaweicloud + +Manage and view Spinnaker configuration for the huaweicloud provider + +#### Usage +``` +hal config provider huaweicloud [parameters] [subcommands] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + +#### Subcommands + * `account`: Manage and view Spinnaker configuration for the huaweicloud provider's account + * `bakery`: Manage and view Spinnaker configuration for the huaweicloud provider's image bakery configuration. + * `disable`: Set the huaweicloud provider as disabled + * `enable`: Set the huaweicloud provider as enabled + +--- +## hal config provider huaweicloud account + +Manage and view Spinnaker configuration for the huaweicloud provider's account + +#### Usage +``` +hal config provider huaweicloud account ACCOUNT [parameters] [subcommands] +``` + +#### Parameters +`ACCOUNT`: The name of the account to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + +#### Subcommands + * `add`: Add an account to the huaweicloud provider. + * `delete`: Delete a specific huaweicloud account by name. + * `edit`: Edit an account in the huaweicloud provider. + * `get`: Get the specified account details for the huaweicloud provider. + * `list`: List the account names for the huaweicloud provider. + +--- +## hal config provider huaweicloud account add + +Add an account to the huaweicloud provider. + +#### Usage +``` +hal config provider huaweicloud account add ACCOUNT [parameters] +``` + +#### Parameters +`ACCOUNT`: The name of the account to operate on. + * `--account-type`: The type of account. + * `--auth-url`: (*Required*) The auth url of cloud. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--domain-name`: (*Required*) The domain name of the cloud. + * `--environment`: The environment name for the account. Many accounts can share the same environment (e.g. dev, test, prod) + * `--insecure`: (*Default*: `false`) Disable certificate validation on SSL connections. Needed if certificates are self signed. Default false. + * `--no-validate`: (*Default*: `false`) Skip validation. + * `--password`: (*Required*) (*Sensitive data* - user will be prompted on standard input) (Sensitive data - user will be prompted on standard input) The password used to access cloud. + * `--project-name`: (*Required*) The name of the project within the cloud. + * `--provider-version`: Some providers support multiple versions/release tracks. This allows you to pick the version of the provider (not the resources it manages) to run within Spinnaker. + * `--read-permissions`: (*Default*: `[]`) A user must have at least one of these roles in order to view this account's cloud resources. + * `--regions`: (*Default*: `[]`) (*Required*) The region(s) of the cloud. + * `--required-group-membership`: (*Default*: `[]`) A user must be a member of at least one specified group in order to make changes to this account's cloud resources. + * `--username`: (*Required*) The username used to access cloud. + * `--write-permissions`: (*Default*: `[]`) A user must have at least one of these roles in order to make changes to this account's cloud resources. + + +--- +## hal config provider huaweicloud account delete + +Delete a specific huaweicloud account by name. + +#### Usage +``` +hal config provider huaweicloud account delete ACCOUNT [parameters] +``` + +#### Parameters +`ACCOUNT`: The name of the account to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud account edit + +Edit an account in the huaweicloud provider. + +#### Usage +``` +hal config provider huaweicloud account edit ACCOUNT [parameters] +``` + +#### Parameters +`ACCOUNT`: The name of the account to operate on. + * `--account-type`: The type of account. + * `--add-read-permission`: Add this permission to the list of read permissions. + * `--add-region`: Add this region to the list of managed regions. + * `--add-required-group-membership`: Add this group to the list of required group memberships. + * `--add-write-permission`: Add this permission to the list of write permissions. + * `--auth-url`: The auth url of cloud. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--domain-name`: The domain name of the cloud. + * `--environment`: The environment name for the account. Many accounts can share the same environment (e.g. dev, test, prod) + * `--insecure`: Disable certificate validation on SSL connections. Needed if certificates are self signed. Default false. + * `--no-validate`: (*Default*: `false`) Skip validation. + * `--password`: (*Sensitive data* - user will be prompted on standard input) (Sensitive data - user will be prompted on standard input) The password used to access cloud. + * `--project-name`: The name of the project within the cloud. + * `--provider-version`: Some providers support multiple versions/release tracks. This allows you to pick the version of the provider (not the resources it manages) to run within Spinnaker. + * `--read-permissions`: A user must have at least one of these roles in order to view this account's cloud resources. + * `--regions`: (*Default*: `[]`) The region(s) of the cloud. + * `--remove-read-permission`: Remove this permission from the list of read permissions. + * `--remove-region`: Remove this region from the list of managed regions. + * `--remove-required-group-membership`: Remove this group from the list of required group memberships. + * `--remove-write-permission`: Remove this permission to from list of write permissions. + * `--required-group-membership`: A user must be a member of at least one specified group in order to make changes to this account's cloud resources. + * `--username`: The username used to access cloud. + * `--write-permissions`: A user must have at least one of these roles in order to make changes to this account's cloud resources. + + +--- +## hal config provider huaweicloud account get + +Get the specified account details for the huaweicloud provider. + +#### Usage +``` +hal config provider huaweicloud account get ACCOUNT [parameters] +``` + +#### Parameters +`ACCOUNT`: The name of the account to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud account list + +List the account names for the huaweicloud provider. + +#### Usage +``` +hal config provider huaweicloud account list [parameters] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud bakery + +Manage and view Spinnaker configuration for the huaweicloud provider's image bakery configuration. + +#### Usage +``` +hal config provider huaweicloud bakery [parameters] [subcommands] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + +#### Subcommands + * `base-image`: Manage and view Spinnaker configuration for the huaweicloud provider's base image. + * `edit`: Edit the huaweicloud provider's bakery default options. + +--- +## hal config provider huaweicloud bakery base-image + +Manage and view Spinnaker configuration for the huaweicloud provider's base image. + +#### Usage +``` +hal config provider huaweicloud bakery base-image [parameters] [subcommands] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + +#### Subcommands + * `add`: Add a base image for the huaweicloud provider's bakery. + * `delete`: Delete a specific huaweicloud base image by name. + * `edit`: Edit a base image for the huaweicloud provider's bakery. + * `get`: Get the specified base image details for the huaweicloud provider. + * `list`: List the base image names for the huaweicloud provider. + +--- +## hal config provider huaweicloud bakery base-image add + +Add a base image for the huaweicloud provider's bakery. + +#### Usage +``` +hal config provider huaweicloud bakery base-image add BASE-IMAGE [parameters] +``` + +#### Parameters +`BASE-IMAGE`: The name of the base image to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--detailed-description`: A long description to help human operators identify the image. + * `--eip-type`: (*Required*) The eip type for the baking configuration. See the api doc to get its value + * `--instance-type`: (*Required*) The instance type for the baking configuration. + * `--no-validate`: (*Default*: `false`) Skip validation. + * `--package-type`: This is used to help Spinnaker's bakery download the build artifacts you supply it with. For example, specifying 'deb' indicates that your artifacts will need to be fetched from a debian repository. + * `--region`: (*Required*) The region for the baking configuration. + * `--short-description`: A short description to help human operators identify the image. + * `--source-image-id`: (*Required*) The source image ID for the baking configuration. + * `--ssh-user-name`: (*Required*) The ssh username for the baking configuration. + * `--template-file`: This is the name of the packer template that will be used to bake images from this base image. The template file must be found in this list [https://github.com/spinnaker/rosco/tree/master/rosco-web/config/packer](https://github.com/spinnaker/rosco/tree/master/rosco-web/config/packer), or supplied as described here: [https://spinnaker.io/setup/bakery/](https://spinnaker.io/setup/bakery/) + + +--- +## hal config provider huaweicloud bakery base-image delete + +Delete a specific huaweicloud base image by name. + +#### Usage +``` +hal config provider huaweicloud bakery base-image delete BASE-IMAGE [parameters] +``` + +#### Parameters +`BASE-IMAGE`: The name of the base image to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud bakery base-image edit + +Edit a base image for the huaweicloud provider's bakery. + +#### Usage +``` +hal config provider huaweicloud bakery base-image edit BASE-IMAGE [parameters] +``` + +#### Parameters +`BASE-IMAGE`: The name of the base image to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--detailed-description`: A long description to help human operators identify the image. + * `--eip-type`: The eip type for the baking configuration. See the api doc to get its value + * `--id`: This is the identifier used by your cloud to find this base image. + * `--instance-type`: The instance type for the baking configuration. + * `--no-validate`: (*Default*: `false`) Skip validation. + * `--package-type`: This is used to help Spinnaker's bakery download the build artifacts you supply it with. For example, specifying 'deb' indicates that your artifacts will need to be fetched from a debian repository. + * `--region`: The region for the baking configuration. + * `--short-description`: A short description to help human operators identify the image. + * `--source-image-id`: The source image ID for the baking configuration. + * `--ssh-user-name`: The ssh username for the baking configuration. + * `--template-file`: This is the name of the packer template that will be used to bake images from this base image. The template file must be found in this list [https://github.com/spinnaker/rosco/tree/master/rosco-web/config/packer](https://github.com/spinnaker/rosco/tree/master/rosco-web/config/packer), or supplied as described here: [https://spinnaker.io/setup/bakery/](https://spinnaker.io/setup/bakery/) + + +--- +## hal config provider huaweicloud bakery base-image get + +Get the specified base image details for the huaweicloud provider. + +#### Usage +``` +hal config provider huaweicloud bakery base-image get BASE-IMAGE [parameters] +``` + +#### Parameters +`BASE-IMAGE`: The name of the base image to operate on. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud bakery base-image list + +List the base image names for the huaweicloud provider. + +#### Usage +``` +hal config provider huaweicloud bakery base-image list [parameters] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud bakery edit + +Edit the huaweicloud provider's bakery default options. + +#### Usage +``` +hal config provider huaweicloud bakery edit [parameters] +``` + +#### Parameters + * `--auth-url`: (*Required*) Set the default auth URL your images will be baked in. + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--domain-name`: (*Required*) Set the default domainName your images will be baked in. + * `--eip-bandwidth-size`: (*Required*) Set the bandwidth size of eip your images will be baked in. + * `--insecure`: (*Required*) The security setting (true/false) for connecting to the HuaweiCloud account. + * `--no-validate`: (*Default*: `false`) Skip validation. + * `--password`: (*Required*) (*Sensitive data* - user will be prompted on standard input) Set the default password your images will be baked with. + * `--project-name`: (*Required*) Set the default project name your images will be baked in. + * `--security-group`: (*Required*) Set the default security group your images will be baked in. + * `--subnet-id`: (*Required*) Set the subnet your images will be baked in. + * `--template-file`: This is the name of the packer template that will be used to bake images from this base image. The template file must be found in this list [https://github.com/spinnaker/rosco/tree/master/rosco-web/config/packer](https://github.com/spinnaker/rosco/tree/master/rosco-web/config/packer), or supplied as described here: [https://spinnaker.io/setup/bakery/](https://spinnaker.io/setup/bakery/) + * `--username`: (*Required*) Set the default username your images will be baked with. + * `--vpc-id`: (*Required*) Set the vpc your images will be baked in. + + +--- +## hal config provider huaweicloud disable + +Set the huaweicloud provider as disabled + +#### Usage +``` +hal config provider huaweicloud disable [parameters] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + +--- +## hal config provider huaweicloud enable + +Set the huaweicloud provider as enabled + +#### Usage +``` +hal config provider huaweicloud enable [parameters] +``` + +#### Parameters + * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment. + * `--no-validate`: (*Default*: `false`) Skip validation. + + --- ## hal config provider kubernetes diff --git a/gradle.properties b/gradle.properties index 6f9eca9f11..cbf1506cbf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -#Mon Oct 28 20:24:06 UTC 2019 +#Mon Dec 09 19:56:50 UTC 2019 enablePublishing=false -korkVersion=6.15.1 +korkVersion=6.22.1 org.gradle.parallel=true diff --git a/halyard-backup/halyard-backup.gradle b/halyard-backup/halyard-backup.gradle index 98a63328a7..f456bb229f 100644 --- a/halyard-backup/halyard-backup.gradle +++ b/halyard-backup/halyard-backup.gradle @@ -16,5 +16,4 @@ dependencies { implementation project(':halyard-config') implementation project(':halyard-core') - implementation project(':halyard-deploy') } diff --git a/halyard-backup/src/main/java/com/netflix/spinnaker/halyard/backup/services/v1/BackupService.java b/halyard-backup/src/main/java/com/netflix/spinnaker/halyard/backup/services/v1/BackupService.java index a310c35539..f88e54f3d1 100644 --- a/halyard-backup/src/main/java/com/netflix/spinnaker/halyard/backup/services/v1/BackupService.java +++ b/halyard-backup/src/main/java/com/netflix/spinnaker/halyard/backup/services/v1/BackupService.java @@ -17,11 +17,15 @@ package com.netflix.spinnaker.halyard.backup.services.v1; +import static com.netflix.spinnaker.halyard.core.problem.v1.Problem.Severity.FATAL; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import com.netflix.spinnaker.halyard.config.config.v1.HalconfigDirectoryStructure; import com.netflix.spinnaker.halyard.config.config.v1.HalconfigParser; import com.netflix.spinnaker.halyard.config.model.v1.node.Halconfig; +import com.netflix.spinnaker.halyard.config.model.v1.node.LocalFile; +import com.netflix.spinnaker.halyard.config.model.v1.node.Node; +import com.netflix.spinnaker.halyard.config.services.v1.FileService; import com.netflix.spinnaker.halyard.core.error.v1.HalException; import com.netflix.spinnaker.halyard.core.problem.v1.Problem; import java.io.BufferedOutputStream; @@ -33,9 +37,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Objects; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveException; @@ -44,6 +48,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -54,23 +59,40 @@ public class BackupService { @Autowired HalconfigDirectoryStructure directoryStructure; + @Autowired private FileService fileService; + static String[] omitPaths = {"service-logs"}; public void restore(String backupTar) { String halconfigDir = directoryStructure.getHalconfigDirectory(); untarHalconfig(halconfigDir, backupTar); + // This is only needed to support old backups where file paths were prefixed with + // {%halconfig-dir%} Halconfig halconfig = halconfigParser.getHalconfig(); - halconfig.makeLocalFilesAbsolute(halconfigDir); + removeHalconfigDirPrefix(halconfig); halconfigParser.saveConfig(); } + /** + * Removes {@link + * com.netflix.spinnaker.halyard.config.model.v1.node.LocalFile#RELATIVE_PATH_PLACEHOLDER} + * instances from a backup. This is held for backwards compatibility reading old backups, new + * backups don't use the prefix and relative file paths are always resolved to hal config home. + * + * @param halconfig instance from backup. + */ + @Deprecated + private void removeHalconfigDirPrefix(Halconfig halconfig) { + makeAbsoluteFilesRelative(halconfig, LocalFile.RELATIVE_PATH_PLACEHOLDER); + } + public String create() { String halconfigDir = directoryStructure.getHalconfigDirectory(); halconfigParser.backupConfig(); Halconfig halconfig = halconfigParser.getHalconfig(); - halconfig.backupLocalFiles(directoryStructure.getBackupConfigDependenciesPath().toString()); - halconfig.makeLocalFilesRelative(halconfigDir); + backupLocalFiles(halconfig, directoryStructure.getBackupConfigDependenciesPath().toString()); + makeAbsoluteFilesRelative(halconfig, halconfigDir); halconfigParser.saveConfig(); SimpleDateFormat dateFormatter = @@ -211,4 +233,74 @@ private void addFileToTar( e); } } + + public List backupLocalFiles(Node node, String outputPath) { + List files = new ArrayList<>(); + + Consumer fileFinder = + n -> + files.addAll( + n.localFiles().stream() + .map( + f -> { + try { + Path fPath = fileService.getLocalFilePath(n.getStringFieldValue(f)); + if (fPath == null) { + return null; + } + File fFile = fPath.toFile(); + String fName = fFile.getName().replaceAll("[^-._a-zA-Z0-9]", "-"); + + // Hash the path to uniquely flatten all files into the output directory + Path newName = + Paths.get(outputPath, Math.abs(fPath.hashCode()) + "-" + fName); + File parent = newName.toFile().getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } else if (fFile.getParent() != null + && fFile.getParent().equals(parent.toString())) { + // Don't move paths that are already in the right folder + return fPath.toString(); + } + Files.copy(fPath, newName, REPLACE_EXISTING); + + n.setStringFieldValue(f, newName.toString()); + return newName.toString(); + } catch (IOException e) { + throw new HalException( + FATAL, "Failed to backup user file: " + e.getMessage(), e); + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + node.recursiveConsume(fileFinder); + + return files; + } + + /** + * Changes all file paths of Halconfig and beginning with "root" from being absolute, to a + * relative path by removing "root". + * + * @param halconfig instance to transform. + * @param root prefix to remove from the local path. + */ + private void makeAbsoluteFilesRelative(Halconfig halconfig, String root) { + halconfig.recursiveConsume( + n -> + n.localFiles() + .forEach( + field -> { + String fPath = n.getStringFieldValue(field); + if (StringUtils.isEmpty(fPath)) { + return; + } + Path localPath = Paths.get(fPath); + if (localPath.isAbsolute() || localPath.startsWith(root)) { + Path rootPath = Paths.get(root); + Path relativePath = rootPath.relativize(localPath); + n.setStringFieldValue(field, relativePath.toString()); + } + })); + } } diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoAddArtifactAccountCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoAddArtifactAccountCommand.java index cc4652312a..30e020e19d 100644 --- a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoAddArtifactAccountCommand.java +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoAddArtifactAccountCommand.java @@ -69,7 +69,7 @@ public class GitRepoAddArtifactAccountCommand extends AbstractAddArtifactAccount @Parameter( names = "--ssh-trust-unknown-hosts", description = "Setting this to true allows Spinnaker to authenticate with unknown hosts") - private Boolean sshTrustUnknownHosts = null; + private Boolean sshTrustUnknownHosts; @Override protected ArtifactAccount buildArtifactAccount(String accountName) { diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoEditArtifactAccountCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoEditArtifactAccountCommand.java index 8e91e88f92..963fc8025f 100644 --- a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoEditArtifactAccountCommand.java +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/artifacts/gitrepo/GitRepoEditArtifactAccountCommand.java @@ -70,7 +70,7 @@ public class GitRepoEditArtifactAccountCommand @Parameter( names = "--ssh-trust-unknown-hosts", description = "Setting this to true allows Spinnaker to authenticate with unknown hosts") - private Boolean sshTrustUnknownHosts = null; + private Boolean sshTrustUnknownHosts; @Override protected ArtifactAccount editArtifactAccount(GitRepoArtifactAccount account) { @@ -89,7 +89,7 @@ protected ArtifactAccount editArtifactAccount(GitRepoArtifactAccount account) { account.setSshKnownHostsFilePath( isSet(sshKnownHostsFilePath) ? sshKnownHostsFilePath : account.getSshKnownHostsFilePath()); account.setSshTrustUnknownHosts( - isSet(sshTrustUnknownHosts) ? sshTrustUnknownHosts : account.isSshTrustUnknownHosts()); + isSet(sshTrustUnknownHosts) ? sshTrustUnknownHosts : account.getSshTrustUnknownHosts()); return account; } diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/ProviderCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/ProviderCommand.java index 9a41524ab9..0468f662d8 100644 --- a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/ProviderCommand.java +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/ProviderCommand.java @@ -26,6 +26,7 @@ import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.dockerRegistry.DockerRegistryCommand; import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.ecs.EcsCommand; import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.google.GoogleCommand; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud.HuaweiCloudCommand; import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.kubernetes.KubernetesCommand; import lombok.AccessLevel; import lombok.Getter; @@ -52,6 +53,7 @@ public ProviderCommand() { registerSubcommand(new DockerRegistryCommand()); registerSubcommand(new EcsCommand()); registerSubcommand(new GoogleCommand()); + registerSubcommand(new HuaweiCloudCommand()); registerSubcommand(new KubernetesCommand()); registerSubcommand( new com.netflix.spinnaker.halyard.cli.command.v1.config.providers.oracle.OracleCommand()); diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAccountCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAccountCommand.java new file mode 100644 index 0000000000..2604d93739 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAccountCommand.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.AbstractAccountCommand; + +@Parameters(separators = "=") +public class HuaweiCloudAccountCommand extends AbstractAccountCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + public HuaweiCloudAccountCommand() { + super(); + registerSubcommand(new HuaweiCloudAddAccountCommand()); + registerSubcommand(new HuaweiCloudEditAccountCommand()); + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAddAccountCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAddAccountCommand.java new file mode 100644 index 0000000000..ec7701d292 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAddAccountCommand.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.account.AbstractAddAccountCommand; +import com.netflix.spinnaker.halyard.config.model.v1.node.Account; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudAccount; +import java.util.ArrayList; +import java.util.List; + +@Parameters(separators = "=") +public class HuaweiCloudAddAccountCommand extends AbstractAddAccountCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + @Parameter( + names = "--account-type", + description = HuaweiCloudCommandProperties.ACCOUNT_TYPE_DESCRIPTION) + private String accountType; + + @Parameter( + names = "--auth-url", + required = true, + description = HuaweiCloudCommandProperties.AUTH_URL_DESCRIPTION) + private String authUrl; + + @Parameter( + names = "--username", + required = true, + description = HuaweiCloudCommandProperties.USERNAME_DESCRIPTION) + private String username; + + @Parameter( + names = "--password", + required = true, + password = true, + description = HuaweiCloudCommandProperties.PASSWORD_DESCRIPTION) + private String password; + + @Parameter( + names = "--project-name", + required = true, + description = HuaweiCloudCommandProperties.PROJECT_NAME_DESCRIPTION) + private String projectName; + + @Parameter( + names = "--domain-name", + required = true, + description = HuaweiCloudCommandProperties.DOMAIN_NAME_DESCRIPTION) + private String domainName; + + @Parameter( + names = "--regions", + required = true, + variableArity = true, + description = HuaweiCloudCommandProperties.REGIONS_DESCRIPTION) + private List regions = new ArrayList<>(); + + @Parameter(names = "--insecure", description = HuaweiCloudCommandProperties.INSECURE_DESCRIPTION) + private boolean insecure; + + @Override + protected Account buildAccount(String accountName) { + HuaweiCloudAccount account = (HuaweiCloudAccount) new HuaweiCloudAccount().setName(accountName); + account + .setAccountType(accountType) + .setAuthUrl(authUrl) + .setUsername(username) + .setPassword(password) + .setProjectName(projectName) + .setDomainName(domainName) + .setInsecure(insecure) + .setRegions(regions); + + return account; + } + + @Override + protected Account emptyAccount() { + return new HuaweiCloudAccount(); + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAddBaseImageCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAddBaseImageCommand.java new file mode 100644 index 0000000000..a97645969b --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudAddBaseImageCommand.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.bakery.AbstractAddBaseImageCommand; +import com.netflix.spinnaker.halyard.config.model.v1.node.BaseImage; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBaseImage; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBaseImage.HuaweiCloudVirtualizationSettings; +import java.util.ArrayList; + +@Parameters(separators = "=") +public class HuaweiCloudAddBaseImageCommand extends AbstractAddBaseImageCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + @Parameter( + names = "--region", + required = true, + description = HuaweiCloudCommandProperties.REGION_DESCRIPTION) + private String region; + + @Parameter( + names = "--instance-type", + required = true, + description = HuaweiCloudCommandProperties.INSTANCE_TYPE_DESCRIPTION) + private String instanceType; + + @Parameter( + names = "--source-image-id", + required = true, + description = HuaweiCloudCommandProperties.SOURCE_IMAGE_ID_DESCRIPTION) + private String sourceImageId; + + @Parameter( + names = "--ssh-user-name", + required = true, + description = HuaweiCloudCommandProperties.SSH_USER_NAME_DESCRIPTION) + private String sshUserName; + + @Parameter( + names = "--eip-type", + required = true, + description = HuaweiCloudCommandProperties.EIP_TYPE_DESCRIPTION) + private String eipType; + + @Override + protected BaseImage buildBaseImage(String baseImageId) { + HuaweiCloudVirtualizationSettings virtualizationSettings = + new HuaweiCloudVirtualizationSettings(); + + virtualizationSettings.setSourceImageId(sourceImageId); + virtualizationSettings.setRegion(region); + virtualizationSettings.setInstanceType(instanceType); + virtualizationSettings.setSshUserName(sshUserName); + virtualizationSettings.setEipType(eipType); + + HuaweiCloudBaseImage baseImage = new HuaweiCloudBaseImage(); + baseImage.setBaseImage(new HuaweiCloudBaseImage.HuaweiCloudImageSettings()); + baseImage.setVirtualizationSettings( + new ArrayList() { + { + add(virtualizationSettings); + } + }); + + return baseImage; + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudBakeryCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudBakeryCommand.java new file mode 100644 index 0000000000..35ef0ed5c1 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudBakeryCommand.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.AbstractBakeryCommand; + +@Parameters(separators = "=") +public class HuaweiCloudBakeryCommand extends AbstractBakeryCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + public HuaweiCloudBakeryCommand() { + super(); + registerSubcommand(new HuaweiCloudEditBakeryDefaultsCommand()); + registerSubcommand(new HuaweiCloudBaseImageCommand()); + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudBaseImageCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudBaseImageCommand.java new file mode 100644 index 0000000000..e203f02639 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudBaseImageCommand.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.bakery.AbstractBaseImageCommand; + +@Parameters(separators = "=") +public class HuaweiCloudBaseImageCommand extends AbstractBaseImageCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + public HuaweiCloudBaseImageCommand() { + super(); + registerSubcommand(new HuaweiCloudAddBaseImageCommand()); + registerSubcommand(new HuaweiCloudEditBaseImageCommand()); + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudCommand.java new file mode 100644 index 0000000000..09953a5782 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudCommand.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.AbstractNamedProviderCommand; + +@Parameters(separators = "=") +public class HuaweiCloudCommand extends AbstractNamedProviderCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + public HuaweiCloudCommand() { + super(); + registerSubcommand(new HuaweiCloudAccountCommand()); + registerSubcommand(new HuaweiCloudBakeryCommand()); + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudCommandProperties.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudCommandProperties.java new file mode 100644 index 0000000000..5d9c5fe05d --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudCommandProperties.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +public class HuaweiCloudCommandProperties { + static final String ACCOUNT_TYPE_DESCRIPTION = "The type of account."; + + static final String AUTH_URL_DESCRIPTION = "The auth url of cloud."; + + static final String USERNAME_DESCRIPTION = "The username used to access cloud."; + + static final String PASSWORD_DESCRIPTION = + "(Sensitive data - user will be prompted on standard input) The password used to access cloud."; + + static final String PROJECT_NAME_DESCRIPTION = "The name of the project within the cloud."; + + static final String DOMAIN_NAME_DESCRIPTION = "The domain name of the cloud."; + + static final String REGIONS_DESCRIPTION = "The region(s) of the cloud."; + + static final String INSECURE_DESCRIPTION = + "Disable certificate validation on SSL connections. Needed if certificates are self signed. Default false."; + + static final String INSTANCE_TYPE_DESCRIPTION = "The instance type for the baking configuration."; + + static final String SOURCE_IMAGE_ID_DESCRIPTION = + "The source image ID for the baking configuration."; + + static final String SSH_USER_NAME_DESCRIPTION = "The ssh username for the baking configuration."; + + static final String REGION_DESCRIPTION = "The region for the baking configuration."; + + static final String EIP_TYPE_DESCRIPTION = + "The eip type for the baking configuration. See the api doc to get its value"; +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditAccountCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditAccountCommand.java new file mode 100644 index 0000000000..864826c0e5 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditAccountCommand.java @@ -0,0 +1,99 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.account.AbstractEditAccountCommand; +import com.netflix.spinnaker.halyard.config.model.v1.node.Account; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudAccount; +import java.util.ArrayList; +import java.util.List; + +@Parameters(separators = "=") +public class HuaweiCloudEditAccountCommand extends AbstractEditAccountCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + @Parameter( + names = "--account-type", + description = HuaweiCloudCommandProperties.ACCOUNT_TYPE_DESCRIPTION) + private String accountType; + + @Parameter(names = "--auth-url", description = HuaweiCloudCommandProperties.AUTH_URL_DESCRIPTION) + private String authUrl; + + @Parameter(names = "--username", description = HuaweiCloudCommandProperties.USERNAME_DESCRIPTION) + private String username; + + @Parameter( + names = "--password", + password = true, + description = HuaweiCloudCommandProperties.PASSWORD_DESCRIPTION) + private String password; + + @Parameter( + names = "--project-name", + description = HuaweiCloudCommandProperties.PROJECT_NAME_DESCRIPTION) + private String projectName; + + @Parameter( + names = "--domain-name", + description = HuaweiCloudCommandProperties.DOMAIN_NAME_DESCRIPTION) + private String domainName; + + @Parameter( + names = "--regions", + variableArity = true, + description = HuaweiCloudCommandProperties.REGIONS_DESCRIPTION) + private List regions = new ArrayList<>(); + + @Parameter( + names = "--add-region", + description = "Add this region to the list of managed regions.") + private String addRegion; + + @Parameter( + names = "--remove-region", + description = "Remove this region from the list of managed regions.") + private String removeRegion; + + @Parameter(names = "--insecure", description = HuaweiCloudCommandProperties.INSECURE_DESCRIPTION) + private Boolean insecure; + + @Override + protected Account editAccount(HuaweiCloudAccount account) { + account.setAccountType(isSet(accountType) ? accountType : account.getAccountType()); + account.setAuthUrl(isSet(authUrl) ? authUrl : account.getAuthUrl()); + account.setUsername(isSet(username) ? username : account.getUsername()); + account.setPassword(isSet(password) ? password : account.getPassword()); + account.setProjectName(isSet(projectName) ? projectName : account.getProjectName()); + account.setDomainName(isSet(domainName) ? domainName : account.getDomainName()); + account.setInsecure(isSet(insecure) ? insecure : account.getInsecure()); + + try { + List existingRegions = account.getRegions(); + List newRegions = updateStringList(existingRegions, regions, addRegion, removeRegion); + account.setRegions(newRegions); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Set either --regions or --[add/remove]-region"); + } + + return account; + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditBakeryDefaultsCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditBakeryDefaultsCommand.java new file mode 100644 index 0000000000..0a9cf232f6 --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditBakeryDefaultsCommand.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.bakery.AbstractEditBakeryDefaultsCommand; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.bakery.BakeryCommandProperties; +import com.netflix.spinnaker.halyard.config.model.v1.node.BakeryDefaults; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBakeryDefaults; + +@Parameters(separators = "=") +public class HuaweiCloudEditBakeryDefaultsCommand + extends AbstractEditBakeryDefaultsCommand { + protected String getProviderName() { + return "huaweicloud"; + } + + @Parameter( + names = "--auth-url", + required = true, + description = "Set the default auth URL your images will be baked in.") + private String authUrl; + + @Parameter( + names = "--username", + required = true, + description = "Set the default username your images will be baked with.") + private String username; + + @Parameter( + names = "--password", + required = true, + password = true, + description = "Set the default password your images will be baked with.") + private String password; + + @Parameter( + names = "--project-name", + required = true, + description = "Set the default project name your images will be baked in.") + private String projectName; + + @Parameter( + names = "--domain-name", + required = true, + description = "Set the default domainName your images will be baked in.") + private String domainName; + + @Parameter( + names = "--insecure", + required = true, + arity = 1, + description = "The security setting (true/false) for connecting to the HuaweiCloud account.") + private Boolean insecure; + + @Parameter( + names = "--vpc-id", + required = true, + description = "Set the vpc your images will be baked in.") + private String vpcId; + + @Parameter( + names = "--subnet-id", + required = true, + description = "Set the subnet your images will be baked in.") + private String subnetId; + + @Parameter( + names = "--eip-bandwidth-size", + required = true, + description = "Set the bandwidth size of eip your images will be baked in.") + private Integer eipBandwidthSize; + + @Parameter( + names = "--security-group", + required = true, + description = "Set the default security group your images will be baked in.") + private String securityGroup; + + @Parameter( + names = "--template-file", + description = BakeryCommandProperties.TEMPLATE_FILE_DESCRIPTION) + private String templateFile; + + @Override + protected BakeryDefaults editBakeryDefaults(HuaweiCloudBakeryDefaults bakeryDefaults) { + bakeryDefaults.setAuthUrl(isSet(authUrl) ? authUrl : bakeryDefaults.getAuthUrl()); + bakeryDefaults.setUsername(isSet(username) ? username : bakeryDefaults.getUsername()); + bakeryDefaults.setPassword(isSet(password) ? password : bakeryDefaults.getPassword()); + bakeryDefaults.setProjectName( + isSet(projectName) ? projectName : bakeryDefaults.getProjectName()); + bakeryDefaults.setDomainName(isSet(domainName) ? domainName : bakeryDefaults.getDomainName()); + bakeryDefaults.setInsecure(isSet(insecure) ? insecure : bakeryDefaults.getInsecure()); + bakeryDefaults.setVpcId(isSet(vpcId) ? vpcId : bakeryDefaults.getVpcId()); + bakeryDefaults.setSubnetId(isSet(subnetId) ? subnetId : bakeryDefaults.getSubnetId()); + bakeryDefaults.setEipBandwidthSize( + isSet(eipBandwidthSize) ? eipBandwidthSize : bakeryDefaults.getEipBandwidthSize()); + bakeryDefaults.setSecurityGroup( + isSet(securityGroup) ? securityGroup : bakeryDefaults.getSecurityGroup()); + bakeryDefaults.setTemplateFile( + isSet(templateFile) ? templateFile : bakeryDefaults.getTemplateFile()); + + return bakeryDefaults; + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditBaseImageCommand.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditBaseImageCommand.java new file mode 100644 index 0000000000..b547f90f7c --- /dev/null +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/providers/huaweicloud/HuaweiCloudEditBaseImageCommand.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.cli.command.v1.config.providers.huaweicloud; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.netflix.spinnaker.halyard.cli.command.v1.config.providers.bakery.AbstractEditBaseImageCommand; +import com.netflix.spinnaker.halyard.config.model.v1.node.BaseImage; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBaseImage; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBaseImage.HuaweiCloudVirtualizationSettings; + +@Parameters(separators = "=") +public class HuaweiCloudEditBaseImageCommand + extends AbstractEditBaseImageCommand { + @Override + protected String getProviderName() { + return "huaweicloud"; + } + + @Parameter(names = "--region", description = HuaweiCloudCommandProperties.REGION_DESCRIPTION) + private String region; + + @Parameter( + names = "--instance-type", + description = HuaweiCloudCommandProperties.INSTANCE_TYPE_DESCRIPTION) + private String instanceType; + + @Parameter( + names = "--source-image-id", + description = HuaweiCloudCommandProperties.SOURCE_IMAGE_ID_DESCRIPTION) + private String sourceImageId; + + @Parameter( + names = "--ssh-user-name", + description = HuaweiCloudCommandProperties.SSH_USER_NAME_DESCRIPTION) + private String sshUserName; + + @Parameter(names = "--eip-type", description = HuaweiCloudCommandProperties.EIP_TYPE_DESCRIPTION) + private String eipType; + + @Override + protected BaseImage editBaseImage(HuaweiCloudBaseImage baseImage) { + HuaweiCloudVirtualizationSettings virtualizationSettings = + baseImage.getVirtualizationSettings().get(0); + + virtualizationSettings.setRegion(isSet(region) ? region : virtualizationSettings.getRegion()); + virtualizationSettings.setSourceImageId( + isSet(sourceImageId) ? sourceImageId : virtualizationSettings.getSourceImageId()); + virtualizationSettings.setInstanceType( + isSet(instanceType) ? instanceType : virtualizationSettings.getInstanceType()); + virtualizationSettings.setSshUserName( + isSet(sshUserName) ? sshUserName : virtualizationSettings.getSshUserName()); + virtualizationSettings.setEipType( + isSet(eipType) ? eipType : virtualizationSettings.getEipType()); + return baseImage; + } +} diff --git a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/services/v1/Daemon.java b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/services/v1/Daemon.java index 3f754ea244..f369e3da62 100644 --- a/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/services/v1/Daemon.java +++ b/halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/services/v1/Daemon.java @@ -1369,13 +1369,6 @@ public static Supplier setPluginDownloadingEnableDisable( }; } - public static Supplier getManifest( - String deploymentName, String serviceName, String apiGroupVersion) { - return () -> - ResponseUnwrapper.get( - getService().getManifest(deploymentName, serviceName, apiGroupVersion)); - } - public static Supplier getTelemetry(String deploymentName, boolean validate) { return () -> { Object rawTelemetry = @@ -1400,6 +1393,13 @@ public static Supplier setTelemetry( }; } + public static Supplier getManifest( + String deploymentName, String serviceName, String apiGroupVersion) { + return () -> + ResponseUnwrapper.get( + getService().getManifest(deploymentName, serviceName, apiGroupVersion)); + } + private static DaemonService service; private static ObjectMapper objectMapper; diff --git a/halyard-config/halyard-config.gradle b/halyard-config/halyard-config.gradle index 64ced41982..99d2054c60 100644 --- a/halyard-config/halyard-config.gradle +++ b/halyard-config/halyard-config.gradle @@ -17,6 +17,7 @@ dependencies { implementation 'com.netflix.spinnaker.kork:kork-secrets-aws' implementation 'com.netflix.spinnaker.kork:kork-secrets-gcp' implementation "com.netflix.spinnaker.kork:kork-web" + implementation "com.netflix.spinnaker.kork:kork-config" implementation 'com.amazonaws:aws-java-sdk-core:1.11.534' implementation 'com.amazonaws:aws-java-sdk-s3:1.11.534' implementation 'com.google.apis:google-api-services-compute' diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParser.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParser.java index 5194ad505c..3708caadb3 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParser.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParser.java @@ -77,9 +77,7 @@ public class HalconfigParser { */ Halconfig parseHalconfig(InputStream is) throws IllegalArgumentException { Object obj = yamlParser.load(is); - Halconfig halconfig = objectMapper.convertValue(obj, Halconfig.class); - halconfig.setPrefixToRelativeFiles(halconfigDirectoryStructure.getRelativeFilesHome()); - return halconfig; + return objectMapper.convertValue(obj, Halconfig.class); } /** @@ -233,8 +231,6 @@ private void saveConfigTo(Path path) { AtomicFileWriter writer = null; try { - local.removePrefixFromUnchangedRelativeFiles( - halconfigDirectoryStructure.getRelativeFilesHome()); writer = new AtomicFileWriter(path); writer.write(yamlParser.dump(objectMapper.convertValue(local, Map.class))); writer.commit(); diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/artifacts/gitrepo/GitRepoArtifactAccount.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/artifacts/gitrepo/GitRepoArtifactAccount.java index 902abbf1d8..a4b2ef34d5 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/artifacts/gitrepo/GitRepoArtifactAccount.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/artifacts/gitrepo/GitRepoArtifactAccount.java @@ -37,5 +37,5 @@ public class GitRepoArtifactAccount extends ArtifactAccount { @LocalFile @SecretFile String sshPrivateKeyFilePath; @Secret String sshPrivateKeyPassphrase; @LocalFile @SecretFile String sshKnownHostsFilePath; - boolean sshTrustUnknownHosts; + Boolean sshTrustUnknownHosts = false; } diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/LocalFile.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/LocalFile.java index 4af148385a..4ab181f62d 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/LocalFile.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/LocalFile.java @@ -28,5 +28,10 @@ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) public @interface LocalFile { - String RELATIVE_PATH_PLACEHOLDER = "{%halconfig-dir%}"; + /** + * File path prefix used in old backup tarballs for local file references. Relative file paths are + * now supported and resolved to hal config home automatically, so this prefix is not needed + * anymore. + */ + @Deprecated String RELATIVE_PATH_PLACEHOLDER = "{%halconfig-dir%}"; } diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Node.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Node.java index 5c309c39c5..302beff6f1 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Node.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Node.java @@ -18,7 +18,6 @@ import static com.netflix.spinnaker.halyard.config.model.v1.node.NodeDiff.ChangeType.*; import static com.netflix.spinnaker.halyard.core.problem.v1.Problem.Severity.FATAL; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; @@ -32,7 +31,6 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -418,191 +416,37 @@ public NodeDiff diff(Node other) { return result; } - public void removePrefixFromUnchangedRelativeFiles(String prefix) { - Consumer fileFinder = - n -> - n.localFiles() - .forEach( - f -> { - try { - f.setAccessible(true); - if (!relativeFileReferences.containsKey(f)) { - return; - } - - String currentPath = (String) f.get(n); - if (StringUtils.isEmpty(currentPath)) { - return; - } - - // restore original relative path - String relativePath = relativeFileReferences.get(f); - String absolutePath = prefix + File.separator + relativePath; - if (currentPath.equals(absolutePath)) { - f.set(n, relativePath); - relativeFileReferences.remove(f); - } - } catch (IllegalAccessException e) { - throw new RuntimeException( - "Failed to get local files for node " + n.getNodeName(), e); - } finally { - f.setAccessible(false); - } - }); - - recursiveConsume(fileFinder); - } - - public void setPrefixToRelativeFiles(String prefix) { - Consumer fileFinder = - n -> - n.localFiles() - .forEach( - f -> { - try { - f.setAccessible(true); - String fPath = (String) f.get(n); - if (StringUtils.isEmpty(fPath)) { - return; - } - if (fPath.startsWith(LocalFile.RELATIVE_PATH_PLACEHOLDER) - || Paths.get(fPath).isAbsolute()) { - return; - } - - Path absolutePath = Paths.get(prefix + File.separator + fPath).normalize(); - if (!absolutePath.startsWith(prefix)) { - throw new HalException( - FATAL, - "Error resolving file path '" - + fPath - + "': Relative file paths must resolve to files inside " - + prefix); - } - - // backup relative path - relativeFileReferences.put(f, fPath); - f.set(n, absolutePath.toString()); - } catch (IllegalAccessException e) { - throw new RuntimeException( - "Failed to get local files for node " + n.getNodeName(), e); - } finally { - f.setAccessible(false); - } - }); - - recursiveConsume(fileFinder); - } - - private void swapLocalFilePrefixes(String to, String from) { - Consumer fileFinder = - n -> - n.localFiles() - .forEach( - f -> { - try { - f.setAccessible(true); - String fPath = (String) f.get(n); - if (fPath == null) { - return; - } - - if (fPath.startsWith(to)) { - log.info( - "File " + f.getName() + " was already in correct format " + fPath); - return; - } - - if (!fPath.startsWith(from)) { - throw new HalException( - FATAL, - "Local file: " - + fPath - + " has incorrect prefix - must match " - + from); - } - - fPath = to + fPath.substring(from.length()); - f.set(n, fPath); - } catch (IllegalAccessException e) { - throw new RuntimeException( - "Failed to get local files for node " + n.getNodeName(), e); - } finally { - f.setAccessible(false); - } - }); - - recursiveConsume(fileFinder); - } - - public void makeLocalFilesRelative(String halconfigPath) { - swapLocalFilePrefixes(LocalFile.RELATIVE_PATH_PLACEHOLDER, halconfigPath); - } - - public void makeLocalFilesAbsolute(String halconfigPath) { - swapLocalFilePrefixes(halconfigPath, LocalFile.RELATIVE_PATH_PLACEHOLDER); + public String getStringFieldValue(Field f) { + try { + f.setAccessible(true); + String value = (String) f.get(this); + if (StringUtils.isNotEmpty(value)) { + return value; + } + // if no value in field, try using getter + try { + return (String) + this.getClass().getMethod("get" + StringUtils.capitalize(f.getName())).invoke(this); + } catch (NoSuchMethodException | InvocationTargetException ignored) { + return null; + } + } catch (IllegalAccessException e) { + return null; + } finally { + f.setAccessible(false); + } } - public List backupLocalFiles(String outputPath) { - List files = new ArrayList<>(); - - Consumer fileFinder = - n -> - files.addAll( - n.localFiles().stream() - .map( - f -> { - try { - f.setAccessible(true); - String fPath = (String) f.get(n); - if (fPath == null) { - try { - fPath = - (String) - n.getClass() - .getMethod("get" + StringUtils.capitalize(f.getName())) - .invoke(n); - } catch (NoSuchMethodException | InvocationTargetException ignored) { - } - } - - if (fPath == null) { - return null; - } - - File fFile = new File(fPath); - String fName = fFile.getName().replaceAll("[^-._a-zA-Z0-9]", "-"); - - // Hash the path to uniquely flatten all files into the output directory - Path newName = - Paths.get(outputPath, Math.abs(fPath.hashCode()) + "-" + fName); - File parent = newName.toFile().getParentFile(); - if (!parent.exists()) { - parent.mkdirs(); - } else if (fFile.getParent() != null - && fFile.getParent().equals(parent.toString())) { - // Don't move paths that are already in the right folder - return fPath; - } - Files.copy(Paths.get(fPath), newName, REPLACE_EXISTING); - - f.set(n, newName.toString()); - return newName.toString(); - } catch (IllegalAccessException e) { - throw new RuntimeException( - "Failed to get local files for node " + n.getNodeName(), e); - } catch (IOException e) { - throw new HalException( - FATAL, "Failed to backup user file: " + e.getMessage(), e); - } finally { - f.setAccessible(false); - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList())); - recursiveConsume(fileFinder); - - return files; + public void setStringFieldValue(Field f, String value) { + try { + f.setAccessible(true); + f.set(this, value); + } catch (IllegalAccessException e) { + throw new HalException( + FATAL, String.format("Unable to set value %s to field %s", value, f.getName())); + } finally { + f.setAccessible(false); + } } public T cloneNode(Class tClass) { diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Plugins.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Plugins.java index d0d1364a48..c9321893fb 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Plugins.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Plugins.java @@ -27,7 +27,7 @@ @Data @EqualsAndHashCode(callSuper = false) -public class Plugins extends Node implements HasEnabled { +public class Plugins extends Node { private List plugins = new ArrayList<>(); private boolean enabled; private boolean downloadingEnabled; diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Provider.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Provider.java index bebd8c0a07..f768166f21 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Provider.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Provider.java @@ -97,6 +97,7 @@ public enum ProviderType { DCOS("dcos"), DOCKERREGISTRY("dockerRegistry"), GOOGLE("google", "gce"), + HUAWEICLOUD("huaweicloud"), KUBERNETES("kubernetes"), ORACLE("oracle"), ORACLEBMCS("oraclebmcs"); // obsolete, replaced by ORACLE diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Providers.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Providers.java index a10737f2a3..24fbeb6bd4 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Providers.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Providers.java @@ -27,6 +27,7 @@ import com.netflix.spinnaker.halyard.config.model.v1.providers.dockerRegistry.DockerRegistryProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.ecs.EcsProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.google.GoogleProvider; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.oracle.OracleBMCSProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.oracle.OracleProvider; @@ -49,6 +50,7 @@ public class Providers extends Node implements Cloneable { DCOSProvider dcos = new DCOSProvider(); DockerRegistryProvider dockerRegistry = new DockerRegistryProvider(); GoogleProvider google = new GoogleProvider(); + HuaweiCloudProvider huaweicloud = new HuaweiCloudProvider(); KubernetesProvider kubernetes = new KubernetesProvider(); @JsonProperty(access = Access.WRITE_ONLY) diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Telemetry.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Telemetry.java index 9f423fc384..5d25fe71a8 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Telemetry.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Telemetry.java @@ -22,7 +22,7 @@ @Data @EqualsAndHashCode(callSuper = false) -public class Telemetry extends Node implements HasEnabled { +public class Telemetry extends Node { public static String DEFAULT_TELEMETRY_ENDPOINT = "https://stats.spinnaker.io"; @@ -31,7 +31,7 @@ public String getNodeName() { return "telemetry"; } - private boolean enabled = false; + private Boolean enabled = false; private String endpoint = DEFAULT_TELEMETRY_ENDPOINT; private String instanceId = new ULID().nextULID(); private String spinnakerVersion; diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Validator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Validator.java index ea33040d63..a57815067e 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Validator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/node/Validator.java @@ -16,21 +16,57 @@ package com.netflix.spinnaker.halyard.config.model.v1.node; -import com.netflix.spinnaker.halyard.config.model.v1.util.ValidatingFileReader; +import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemBuilder; import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; +import com.netflix.spinnaker.halyard.config.services.v1.FileService; +import com.netflix.spinnaker.halyard.core.problem.v1.Problem; import com.netflix.spinnaker.halyard.core.secrets.v1.SecretSessionManager; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Path; import org.springframework.beans.factory.annotation.Autowired; public abstract class Validator { + + private static final String NO_ACCESS_REMEDIATION = + "Halyard is running as user " + + System.getProperty("user.name") + + ". Make sure that user can read the requested file."; + @Autowired protected SecretSessionManager secretSessionManager; + @Autowired private FileService fileService; public abstract void validate(ConfigProblemSetBuilder p, T n); protected String validatingFileDecrypt(ConfigProblemSetBuilder p, String filePath) { - return ValidatingFileReader.contents(p, filePath, secretSessionManager); + byte[] contentBytes = validatingFileDecryptBytes(p, filePath); + if (contentBytes != null) { + return new String(contentBytes); + } + return null; } protected byte[] validatingFileDecryptBytes(ConfigProblemSetBuilder p, String filePath) { - return ValidatingFileReader.contentBytes(p, filePath, secretSessionManager); + try { + return fileService.getFileContentBytes(filePath); + } catch (FileNotFoundException e) { + buildProblem(p, "Cannot find provided path: " + e.getMessage() + ".", e); + } catch (IOException e) { + buildProblem(p, "Failed to read path \"" + filePath + "\".", e); + } + return null; + } + + protected Path validatingFileDecryptPath(String filePath) { + return fileService.getLocalFilePath(filePath); + } + + private void buildProblem(ConfigProblemSetBuilder ps, String message, Exception exception) { + ConfigProblemBuilder problemBuilder = + ps.addProblem(Problem.Severity.FATAL, message + ": " + exception.getMessage() + "."); + + if (exception.getMessage().contains("denied")) { + problemBuilder.setRemediation(NO_ACCESS_REMEDIATION); + } } } diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudAccount.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudAccount.java new file mode 100644 index 0000000000..a1e904f085 --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudAccount.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.Account; +import com.netflix.spinnaker.halyard.config.model.v1.node.Secret; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class HuaweiCloudAccount extends Account { + private String accountType; + private String authUrl; + private String username; + @Secret private String password; + private String projectName; + private String domainName; + private Boolean insecure = false; + private List regions; +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudBakeryDefaults.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudBakeryDefaults.java new file mode 100644 index 0000000000..75c08d28ba --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudBakeryDefaults.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.BakeryDefaults; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class HuaweiCloudBakeryDefaults extends BakeryDefaults { + private String authUrl; + private String username; + private String password; + private String projectName; + private String domainName; + private Boolean insecure; + private String vpcId; + private String subnetId; + private String securityGroup; + private Integer eipBandwidthSize; +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudBaseImage.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudBaseImage.java new file mode 100644 index 0000000000..d91011a570 --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudBaseImage.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.BaseImage; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@ToString +@EqualsAndHashCode(callSuper = true) +public class HuaweiCloudBaseImage + extends BaseImage< + HuaweiCloudBaseImage.HuaweiCloudImageSettings, + List> { + + private HuaweiCloudImageSettings baseImage; + private List virtualizationSettings; + + @EqualsAndHashCode(callSuper = true) + @Data + @ToString(callSuper = true) + public static class HuaweiCloudImageSettings extends BaseImage.ImageSettings {} + + @Data + @ToString + public static class HuaweiCloudVirtualizationSettings { + String region; + String instanceType; + String sourceImageId; + String sshUserName; + String eipType; + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudProvider.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudProvider.java new file mode 100644 index 0000000000..6e845b6886 --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/providers/huaweicloud/HuaweiCloudProvider.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.HasImageProvider; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class HuaweiCloudProvider + extends HasImageProvider implements Cloneable { + @Override + public ProviderType providerType() { + return ProviderType.HUAWEICLOUD; + } + + @Override + public HuaweiCloudBakeryDefaults emptyBakeryDefaults() { + return new HuaweiCloudBakeryDefaults(); + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/util/ValidatingFileReader.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/util/ValidatingFileReader.java index 453e736f0f..8b13789179 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/util/ValidatingFileReader.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/util/ValidatingFileReader.java @@ -1,82 +1 @@ -/* - * Copyright 2017 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.spinnaker.halyard.config.model.v1.util; - -import com.amazonaws.util.IOUtils; -import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemBuilder; -import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; -import com.netflix.spinnaker.halyard.core.problem.v1.Problem; -import com.netflix.spinnaker.halyard.core.secrets.v1.SecretSessionManager; -import com.netflix.spinnaker.kork.secrets.EncryptedSecret; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; - -public class ValidatingFileReader { - - private static final String NO_ACCESS_REMEDIATION = - "Halyard is running as user " - + System.getProperty("user.name") - + ". Make sure that user can read the requested file."; - - public static String contents( - ConfigProblemSetBuilder ps, String path, SecretSessionManager secretSessionManager) { - - byte[] contentBytes = contentBytes(ps, path, secretSessionManager); - if (contentBytes == null) { - return null; - } - return new String(contentBytes); - } - - public static byte[] contentBytes( - ConfigProblemSetBuilder ps, String path, SecretSessionManager secretSessionManager) { - - if (PropertyUtils.isConfigServerResource(path)) { - return null; - } - - if (EncryptedSecret.isEncryptedSecret(path)) { - return secretSessionManager.decryptAsBytes(path); - } - - return readFromLocalFilesystem(ps, path); - } - - private static byte[] readFromLocalFilesystem(ConfigProblemSetBuilder ps, String path) { - try { - return IOUtils.toByteArray(new FileInputStream(path)); - } catch (FileNotFoundException e) { - buildProblem(ps, "Cannot find provided path: " + e.getMessage() + ".", e); - } catch (IOException e) { - buildProblem(ps, "Failed to read path \"" + path + "\".", e); - } - - return null; - } - - private static void buildProblem( - ConfigProblemSetBuilder ps, String message, Exception exception) { - ConfigProblemBuilder problemBuilder = - ps.addProblem(Problem.Severity.FATAL, message + ": " + exception.getMessage() + "."); - - if (exception.getMessage().contains("denied")) { - problemBuilder.setRemediation(ValidatingFileReader.NO_ACCESS_REMEDIATION); - } - } -} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/FileService.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/FileService.java new file mode 100644 index 0000000000..502ce029da --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/FileService.java @@ -0,0 +1,113 @@ +package com.netflix.spinnaker.halyard.config.services.v1; + +import com.amazonaws.util.IOUtils; +import com.netflix.spinnaker.halyard.config.config.v1.HalconfigDirectoryStructure; +import com.netflix.spinnaker.halyard.core.secrets.v1.SecretSessionManager; +import com.netflix.spinnaker.kork.configserver.CloudConfigResourceService; +import com.netflix.spinnaker.kork.configserver.ConfigFileService; +import com.netflix.spinnaker.kork.secrets.EncryptedSecret; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** Service for file operations. */ +@Component +public class FileService { + private final SecretSessionManager secretSessionManager; + private final HalconfigDirectoryStructure halconfigDirectoryStructure; + private final ConfigFileService configFileService; + private final CloudConfigResourceService cloudConfigResourceService; + + @Autowired + public FileService( + SecretSessionManager secretSessionManager, + HalconfigDirectoryStructure halconfigDirectoryStructure, + ConfigFileService configFileService, + CloudConfigResourceService cloudConfigResourceService) { + this.secretSessionManager = secretSessionManager; + this.halconfigDirectoryStructure = halconfigDirectoryStructure; + this.configFileService = configFileService; + this.cloudConfigResourceService = cloudConfigResourceService; + } + + /** + * Returns an absolute file path in the local file system resolved by this file reference, + * retrieving the file from external systems if necessary. + * + * @param fileReference a file reference can be a secret, a config server resource or a path in + * the local file system. + * @return an absolute path to the file, or null if the reference cannot be resolved to a local + * path. + */ + public Path getLocalFilePath(String fileReference) { + if (StringUtils.isEmpty(fileReference)) { + return null; + } + if (CloudConfigResourceService.isCloudConfigResource(fileReference)) { + return Paths.get(cloudConfigResourceService.getLocalPath(fileReference)); + } + if (EncryptedSecret.isEncryptedSecret(fileReference)) { + return Paths.get(secretSessionManager.decryptAsFile(fileReference)); + } + + return absolutePath(fileReference); + } + + /** + * Return the contents of a file as a string. + * + * @param fileReference a file reference can be a secret, a config server resource or a path in + * the local file system. + * @return file contents. + */ + public String getFileContents(String fileReference) throws IOException { + byte[] contentBytes = getFileContentBytes(fileReference); + if (contentBytes == null) { + return null; + } + return new String(contentBytes); + } + + /** + * Return the contents of a file as a byte array. + * + * @param fileReference a file reference can be a secret, a config server resource or a path in + * the local file system. + * @return file contents as bytes. + */ + public byte[] getFileContentBytes(String fileReference) throws IOException { + if (CloudConfigResourceService.isCloudConfigResource(fileReference)) { + String localPath = cloudConfigResourceService.getLocalPath(fileReference); + return configFileService.getContents(localPath).getBytes(); + } + if (EncryptedSecret.isEncryptedSecret(fileReference)) { + return secretSessionManager.decryptAsBytes(fileReference); + } + + return readFromLocalFilesystem(fileReference); + } + + private byte[] readFromLocalFilesystem(String path) throws IOException { + Path absolutePath = absolutePath(path); + if (absolutePath == null) { + throw new IOException( + "Provided path: \"" + path + "\" cannot be resolved to a local absolute path."); + } + return IOUtils.toByteArray(new FileInputStream(absolutePath.toString())); + } + + private Path absolutePath(String path) { + if (StringUtils.isEmpty(path)) { + return null; + } + Path filePath = Paths.get(path); + if (!filePath.isAbsolute()) { + filePath = Paths.get(halconfigDirectoryStructure.getHalconfigDirectory(), path); + } + return filePath; + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/ProviderService.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/ProviderService.java index 8ddabe05b7..788b963e02 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/ProviderService.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/services/v1/ProviderService.java @@ -25,6 +25,7 @@ import com.netflix.spinnaker.halyard.config.model.v1.providers.dcos.DCOSProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.dockerRegistry.DockerRegistryProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.google.GoogleProvider; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.oracle.OracleBMCSProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.oracle.OracleProvider; @@ -126,6 +127,9 @@ public void setProvider(String deploymentName, Provider provider) { case GOOGLE: providers.setGoogle((GoogleProvider) provider); break; + case HUAWEICLOUD: + providers.setHuaweicloud((HuaweiCloudProvider) provider); + break; case KUBERNETES: providers.setKubernetes((KubernetesProvider) provider); break; diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/FieldValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/FieldValidator.java index a945b4424e..26dee0b953 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/FieldValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/FieldValidator.java @@ -16,7 +16,10 @@ package com.netflix.spinnaker.halyard.config.validate.v1; -import com.netflix.spinnaker.halyard.config.model.v1.node.*; +import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration; +import com.netflix.spinnaker.halyard.config.model.v1.node.Node; +import com.netflix.spinnaker.halyard.config.model.v1.node.ValidForSpinnakerVersion; +import com.netflix.spinnaker.halyard.config.model.v1.node.Validator; import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; import com.netflix.spinnaker.halyard.core.problem.v1.Problem; import com.netflix.spinnaker.halyard.core.registry.v1.Versions; diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/canary/google/GoogleCanaryAccountValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/canary/google/GoogleCanaryAccountValidator.java index 6761e92bc2..bcbbdd9074 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/canary/google/GoogleCanaryAccountValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/canary/google/GoogleCanaryAccountValidator.java @@ -27,6 +27,7 @@ import com.netflix.spinnaker.halyard.core.problem.v1.Problem.Severity; import com.netflix.spinnaker.halyard.core.secrets.v1.SecretSessionManager; import com.netflix.spinnaker.halyard.core.tasks.v1.DaemonTaskHandler; +import java.nio.file.Path; import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.commons.lang3.StringUtils; @@ -69,7 +70,7 @@ public void validate(ConfigProblemSetBuilder p, GoogleCanaryAccount n) { return; } - String jsonPath = n.getJsonPath(); + Path jsonPath = validatingFileDecryptPath(n.getJsonPath()); try { StorageService storageService = @@ -78,7 +79,7 @@ public void validate(ConfigProblemSetBuilder p, GoogleCanaryAccount n) { n.getBucketLocation(), n.getRootFolder(), n.getProject(), - jsonPath != null ? secretSessionManager.decryptAsFile(jsonPath) : "", + jsonPath != null ? jsonPath.toString() : "", "halyard", connectTimeoutSec, readTimeoutSec, diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/persistentStorage/GCSValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/persistentStorage/GCSValidator.java index dedea3f633..294d6b9146 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/persistentStorage/GCSValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/persistentStorage/GCSValidator.java @@ -24,6 +24,7 @@ import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; import com.netflix.spinnaker.halyard.config.services.v1.AccountService; import com.netflix.spinnaker.halyard.core.problem.v1.Problem.Severity; +import java.nio.file.Path; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.TaskScheduler; import org.springframework.stereotype.Component; @@ -45,7 +46,7 @@ public class GCSValidator extends Validator { @Override public void validate(ConfigProblemSetBuilder ps, GcsPersistentStore n) { - String jsonPath = n.getJsonPath(); + Path jsonPath = validatingFileDecryptPath(n.getJsonPath()); try { StorageService storageService = new GcsStorageService( @@ -53,7 +54,7 @@ public void validate(ConfigProblemSetBuilder ps, GcsPersistentStore n) { n.getBucketLocation(), n.getRootFolder(), n.getProject(), - jsonPath != null ? secretSessionManager.decryptAsFile(jsonPath) : "", + jsonPath != null ? jsonPath.toString() : "", "halyard", connectTimeoutSec, readTimeoutSec, diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/appengine/AppengineAccountValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/appengine/AppengineAccountValidator.java index 04c1a3f9e5..2250eade02 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/appengine/AppengineAccountValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/appengine/AppengineAccountValidator.java @@ -25,6 +25,7 @@ import com.netflix.spinnaker.halyard.config.model.v1.providers.appengine.AppengineAccount; import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; import com.netflix.spinnaker.halyard.core.problem.v1.Problem.Severity; +import java.nio.file.Path; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -65,7 +66,11 @@ public void validate(ConfigProblemSetBuilder p, AppengineAccount account) { "SSH private key filepath supplied without SSH private key passphrase."); } } else if (hasSshPrivateKeyPassphrase && hasSshPrivateKeyFilePath) { - String sshPrivateKey = validatingFileDecrypt(p, account.getSshPrivateKeyFilePath()); + Path sshPrivateKeyFilePath = validatingFileDecryptPath(account.getSshPrivateKeyFilePath()); + if (sshPrivateKeyFilePath == null) { + return; + } + String sshPrivateKey = validatingFileDecrypt(p, sshPrivateKeyFilePath.toString()); if (sshPrivateKey == null) { return; } else if (sshPrivateKey.isEmpty()) { @@ -74,10 +79,7 @@ public void validate(ConfigProblemSetBuilder p, AppengineAccount account) { try { // Assumes that the public key is sitting next to the private key with the extension // ".pub". - KeyPair keyPair = - KeyPair.load( - new JSch(), - secretSessionManager.decryptAsFile(account.getSshPrivateKeyFilePath())); + KeyPair keyPair = KeyPair.load(new JSch(), sshPrivateKeyFilePath.toString()); boolean decrypted = keyPair.decrypt(secretSessionManager.decrypt(account.getSshPrivateKeyPassphrase())); if (!decrypted) { diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/dockerRegistry/DockerRegistryAccountValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/dockerRegistry/DockerRegistryAccountValidator.java index 90e846ff06..7e51078aa3 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/dockerRegistry/DockerRegistryAccountValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/dockerRegistry/DockerRegistryAccountValidator.java @@ -26,6 +26,7 @@ import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; import com.netflix.spinnaker.halyard.core.problem.v1.Problem.Severity; import java.nio.charset.Charset; +import java.nio.file.Path; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; @@ -120,6 +121,7 @@ public void validate(ConfigProblemSetBuilder p, DockerRegistryAccount n) { DockerRegistryNamedAccountCredentials credentials; try { + Path passwordFilePath = validatingFileDecryptPath(n.getPasswordFile()); credentials = (new DockerRegistryNamedAccountCredentials.Builder()) .accountName(n.getName()) @@ -127,7 +129,7 @@ public void validate(ConfigProblemSetBuilder p, DockerRegistryAccount n) { .email(n.getEmail()) .password(secretSessionManager.decrypt(n.getPassword())) .passwordCommand(n.getPasswordCommand()) - .passwordFile(secretSessionManager.decryptAsFile(n.getPasswordFile())) + .passwordFile(passwordFilePath != null ? passwordFilePath.toString() : null) .dockerconfigFile(n.getDockerconfigFile()) .username(n.getUsername()) .clientTimeoutMillis(n.getClientTimeoutMillis()) diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudAccountValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudAccountValidator.java new file mode 100644 index 0000000000..049b09dcac --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudAccountValidator.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.validate.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.Validator; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudAccount; +import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; +import com.netflix.spinnaker.halyard.core.problem.v1.Problem; +import com.netflix.spinnaker.halyard.core.tasks.v1.DaemonTaskHandler; +import java.util.List; +import lombok.EqualsAndHashCode; +import org.apache.commons.lang3.StringUtils; + +@EqualsAndHashCode(callSuper = false) +public class HuaweiCloudAccountValidator extends Validator { + + @Override + public void validate(ConfigProblemSetBuilder psBuilder, HuaweiCloudAccount account) { + String accountName = account.getNodeName(); + + DaemonTaskHandler.message( + "Validating " + accountName + " with " + HuaweiCloudAccountValidator.class.getSimpleName()); + + String authUrl = account.getAuthUrl(); + if (StringUtils.isEmpty(authUrl)) { + psBuilder.addProblem( + Problem.Severity.ERROR, + String.format("For account: %s, you must provide an identity url.", accountName)); + } + + String username = account.getUsername(); + String password = account.getPassword(); + if (StringUtils.isEmpty(password) || StringUtils.isEmpty(username)) { + psBuilder.addProblem( + Problem.Severity.ERROR, + String.format( + "For account: %s, you must provide both username and password.", accountName)); + } + + String projectName = account.getProjectName(); + if (StringUtils.isEmpty(projectName)) { + psBuilder.addProblem( + Problem.Severity.ERROR, + String.format("For account: %s, you must provide a project name.", accountName)); + } + + String domainName = account.getDomainName(); + if (StringUtils.isEmpty(domainName)) { + psBuilder.addProblem( + Problem.Severity.ERROR, + String.format("For account: %s, you must provide a domain name.", accountName)); + } + + List regions = account.getRegions(); + if (regions.size() == 0) { + psBuilder.addProblem( + Problem.Severity.ERROR, + String.format("For account: %s, you must provide at least one region.", accountName)); + } + + Boolean insecure = account.getInsecure(); + if (insecure) { + psBuilder.addProblem( + Problem.Severity.WARNING, + String.format( + "For account: %s, you've chosen to not validate SSL connections. This setup is not recommended in production deployments.", + accountName)); + } + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudBakeryDefaultsValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudBakeryDefaultsValidator.java new file mode 100644 index 0000000000..913dfc2e6d --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudBakeryDefaultsValidator.java @@ -0,0 +1,102 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.validate.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.Validator; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBakeryDefaults; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBaseImage; +import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; +import com.netflix.spinnaker.halyard.core.problem.v1.Problem; +import com.netflix.spinnaker.halyard.core.tasks.v1.DaemonTaskHandler; +import java.util.List; +import lombok.EqualsAndHashCode; +import org.springframework.util.StringUtils; + +@EqualsAndHashCode(callSuper = false) +public class HuaweiCloudBakeryDefaultsValidator extends Validator { + + @Override + public void validate(ConfigProblemSetBuilder p, HuaweiCloudBakeryDefaults n) { + DaemonTaskHandler.message( + "Validating " + + n.getNodeName() + + " with " + + HuaweiCloudBakeryDefaultsValidator.class.getSimpleName()); + + if ((new HuaweiCloudBakeryDefaults()).equals(n)) { + return; + } + + String authUrl = n.getAuthUrl(); + if (StringUtils.isEmpty(authUrl)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no auth url supplied."); + } + + String username = n.getUsername(); + if (StringUtils.isEmpty(username)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no username supplied."); + } + + String password = n.getPassword(); + if (StringUtils.isEmpty(password)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no password supplied."); + } + + String projectName = n.getProjectName(); + if (StringUtils.isEmpty(projectName)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no project name supplied."); + } + + String domainName = n.getDomainName(); + if (StringUtils.isEmpty(domainName)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no domain name supplied"); + } + + Boolean insecure = n.getInsecure(); + if (insecure) { + p.addProblem( + Problem.Severity.WARNING, + "For bakery defaults: You've chosen to not validate SSL connections. This setup is not recommended in production deployments."); + } + + String vpcId = n.getVpcId(); + if (StringUtils.isEmpty(vpcId)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no vpc id supplied."); + } + + String subnetId = n.getSubnetId(); + if (StringUtils.isEmpty(subnetId)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no subnet id supplied."); + } + + Integer eipBandwidthSize = n.getEipBandwidthSize(); + if (eipBandwidthSize <= 0) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no eip bandwidth size supplied."); + } + + String securityGroup = n.getSecurityGroup(); + if (StringUtils.isEmpty(securityGroup)) { + p.addProblem(Problem.Severity.ERROR, "For bakery defaults: no security group supplied."); + } + + HuaweiCloudBaseImageValidator huaweicloudBaseImageValidator = + new HuaweiCloudBaseImageValidator(); + List baseImages = n.getBaseImages(); + baseImages.forEach( + huaweicloudBaseImage -> huaweicloudBaseImageValidator.validate(p, huaweicloudBaseImage)); + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudBaseImageValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudBaseImageValidator.java new file mode 100644 index 0000000000..ad2921c48b --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudBaseImageValidator.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.validate.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.Validator; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudBaseImage; +import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; +import com.netflix.spinnaker.halyard.core.problem.v1.Problem; +import com.netflix.spinnaker.halyard.core.tasks.v1.DaemonTaskHandler; +import lombok.EqualsAndHashCode; +import org.springframework.util.StringUtils; + +@EqualsAndHashCode(callSuper = false) +public class HuaweiCloudBaseImageValidator extends Validator { + + public void validate(ConfigProblemSetBuilder p, HuaweiCloudBaseImage n) { + String baseImage = n.getNodeName(); + + DaemonTaskHandler.message( + "Validating " + baseImage + " with " + HuaweiCloudBaseImageValidator.class.getSimpleName()); + + HuaweiCloudBaseImage.HuaweiCloudImageSettings imageSetting = n.getBaseImage(); + String packageType = imageSetting.getPackageType(); + if (StringUtils.isEmpty(packageType)) { + p.addProblem( + Problem.Severity.ERROR, + String.format("For base image:%s, no package type supplied.", baseImage)); + } + + HuaweiCloudBaseImage.HuaweiCloudVirtualizationSettings vs = + n.getVirtualizationSettings().get(0); + + String region = vs.getRegion(); + if (StringUtils.isEmpty(region)) { + p.addProblem( + Problem.Severity.ERROR, + String.format("For base image:%s, no region supplied.", baseImage)); + } + + String instanceType = vs.getInstanceType(); + if (StringUtils.isEmpty(instanceType)) { + p.addProblem( + Problem.Severity.ERROR, + String.format("For base image:%s, no instance type supplied.", baseImage)); + } + + String sourceImageId = vs.getSourceImageId(); + if (StringUtils.isEmpty(sourceImageId)) { + p.addProblem( + Problem.Severity.ERROR, + String.format("For base image:%s, no source image id supplied.", baseImage)); + } + + String sshUserName = vs.getSshUserName(); + if (StringUtils.isEmpty(sshUserName)) { + p.addProblem( + Problem.Severity.ERROR, + String.format("For base image:%s, no ssh username supplied.", baseImage)); + } + + String eipType = vs.getEipType(); + if (StringUtils.isEmpty(eipType)) { + p.addProblem( + Problem.Severity.ERROR, + String.format("For base image:%s, no eip type supplied.", baseImage)); + } + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudProviderValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudProviderValidator.java new file mode 100644 index 0000000000..c5f611719c --- /dev/null +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/huaweicloud/HuaweiCloudProviderValidator.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 Huawei Technologies Co.,Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.halyard.config.validate.v1.providers.huaweicloud; + +import com.netflix.spinnaker.halyard.config.model.v1.node.Validator; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudProvider; +import com.netflix.spinnaker.halyard.config.problem.v1.ConfigProblemSetBuilder; +import org.springframework.stereotype.Component; + +@Component +public class HuaweiCloudProviderValidator extends Validator { + + @Override + public void validate(ConfigProblemSetBuilder p, HuaweiCloudProvider n) { + + HuaweiCloudAccountValidator accountValidator = new HuaweiCloudAccountValidator(); + + n.getAccounts().forEach(huaweicloudAccount -> accountValidator.validate(p, huaweicloudAccount)); + + HuaweiCloudBakeryDefaultsValidator bakeryValidator = new HuaweiCloudBakeryDefaultsValidator(); + + bakeryValidator.validate(p, n.getBakeryDefaults()); + } +} diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/kubernetes/KubernetesAccountValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/kubernetes/KubernetesAccountValidator.java index 5966500596..4bc66c8e61 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/kubernetes/KubernetesAccountValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/providers/kubernetes/KubernetesAccountValidator.java @@ -41,6 +41,7 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.internal.KubeConfigUtils; import java.io.IOException; +import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -259,9 +260,10 @@ private void validateKubeconfig(ConfigProblemSetBuilder psBuilder, KubernetesAcc } if (smoketest) { + Path kubeconfigPath = validatingFileDecryptPath(account.getKubeconfigFile()); Config config = KubernetesConfigParser.parse( - secretSessionManager.decryptAsFile(account.getKubeconfigFile()), + kubeconfigPath != null ? kubeconfigPath.toString() : null, context, cluster, user, diff --git a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/security/SamlValidator.java b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/security/SamlValidator.java index 90a47a89e0..0364f961fd 100644 --- a/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/security/SamlValidator.java +++ b/halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/validate/v1/security/SamlValidator.java @@ -47,7 +47,7 @@ public void validate(ConfigProblemSetBuilder p, Saml saml) { if (StringUtils.isNotEmpty(saml.getMetadataLocal())) { try { - new File(new URI("file:" + validatingFileDecrypt(p, saml.getMetadataLocal()))); + new File(new URI("file:" + validatingFileDecryptPath(saml.getMetadataLocal()))); } catch (Exception f) { p.addProblem(Problem.Severity.ERROR, f.getMessage()); } diff --git a/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParserSpec.groovy b/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParserSpec.groovy index 251aee170e..49c311dab0 100644 --- a/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParserSpec.groovy +++ b/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/config/v1/HalconfigParserSpec.groovy @@ -17,7 +17,6 @@ package com.netflix.spinnaker.halyard.config.config.v1 import com.netflix.spinnaker.halyard.config.model.v1.node.Halconfig -import com.netflix.spinnaker.halyard.core.error.v1.HalException import org.yaml.snakeyaml.Yaml import org.yaml.snakeyaml.constructor.SafeConstructor import spock.lang.Specification @@ -29,16 +28,12 @@ class HalconfigParserSpec extends Specification { String HALYARD_VERSION = "0.1.0" String SPINNAKER_VERSION = "1.0.0" String CURRENT_DEPLOYMENT = "my-spinnaker-deployment" - String HALCONFIG_HOME = "/home/spinnaker/.hal" HalconfigParser parser void setup() { parser = new HalconfigParser() parser.yamlParser = new Yaml(new SafeConstructor()) parser.objectMapper = new StrictObjectMapper() - def hcDirStructure = new HalconfigDirectoryStructure() - hcDirStructure.halconfigDirectory = HALCONFIG_HOME - parser.halconfigDirectoryStructure = hcDirStructure } void "Accept minimal config"() { @@ -164,84 +159,4 @@ deploymentConfigurations: // Uncomment the below to implement LDAP. Then fill in the rest of the LDAP properties, one per line. // "ldap" | "enabled" | true } - - void "Adds hal config home to relative file paths"() { - setup: - String config = """ -halyardVersion: 1 -currentDeployment: $CURRENT_DEPLOYMENT -deploymentConfigurations: -- name: $CURRENT_DEPLOYMENT - version: $SPINNAKER_VERSION - providers: - kubernetes: - enabled: true - accounts: - - name: kubernetes - requiredGroupMembership: [] - providerVersion: V2 - permissions: {} - dockerRegistries: [] - configureImagePullSecrets: true - cacheThreads: 1 - namespaces: [] - omitNamespaces: [] - kinds: [] - omitKinds: [] - customResources: [] - cachingPolicies: [] - kubeconfigFile: required-files/kubecfg - oAuthScopes: [] - onlySpinnakerManaged: false - primaryAccount: kubernetes -""" - InputStream stream = new ByteArrayInputStream(config.getBytes(StandardCharsets.UTF_8)) - Halconfig out = null - - when: - out = parser.parseHalconfig(stream) - - then: - out.deploymentConfigurations[0].providers.kubernetes.accounts[0].kubeconfigFile == "$HALCONFIG_HOME/required-files/kubecfg" - } - - void "Throws error when trying to escape hal config home with relative local file paths"() { - setup: - String config = """ -halyardVersion: 1 -currentDeployment: $CURRENT_DEPLOYMENT -deploymentConfigurations: -- name: $CURRENT_DEPLOYMENT - version: $SPINNAKER_VERSION - providers: - kubernetes: - enabled: true - accounts: - - name: kubernetes - requiredGroupMembership: [] - providerVersion: V2 - permissions: {} - dockerRegistries: [] - configureImagePullSecrets: true - cacheThreads: 1 - namespaces: [] - omitNamespaces: [] - kinds: [] - omitKinds: [] - customResources: [] - cachingPolicies: [] - kubeconfigFile: poison/../../.kube/config - oAuthScopes: [] - onlySpinnakerManaged: false - primaryAccount: kubernetes -""" - InputStream stream = new ByteArrayInputStream(config.getBytes(StandardCharsets.UTF_8)) - Halconfig out = null - - when: - out = parser.parseHalconfig(stream) - - then: - thrown HalException - } } diff --git a/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/model/v1/ProvidersSpec.groovy b/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/model/v1/ProvidersSpec.groovy index bb8d6db69e..3594dc0bda 100644 --- a/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/model/v1/ProvidersSpec.groovy +++ b/halyard-config/src/test/groovy/com/netflix/spinnaker/halyard/config/model/v1/ProvidersSpec.groovy @@ -25,6 +25,7 @@ import com.netflix.spinnaker.halyard.config.model.v1.providers.azure.AzureProvid import com.netflix.spinnaker.halyard.config.model.v1.providers.dcos.DCOSProvider import com.netflix.spinnaker.halyard.config.model.v1.providers.dockerRegistry.DockerRegistryProvider import com.netflix.spinnaker.halyard.config.model.v1.providers.google.GoogleProvider +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudProvider import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesProvider import com.netflix.spinnaker.halyard.config.model.v1.providers.oracle.OracleProvider import spock.lang.Specification @@ -62,6 +63,7 @@ class ProvidersSpec extends Specification { DCOSProvider, DockerRegistryProvider, GoogleProvider, + HuaweiCloudProvider, KubernetesProvider, OracleProvider ] diff --git a/halyard-deploy/halyard-deploy.gradle b/halyard-deploy/halyard-deploy.gradle index a5f4c6e050..3d6229801b 100644 --- a/halyard-deploy/halyard-deploy.gradle +++ b/halyard-deploy/halyard-deploy.gradle @@ -26,6 +26,7 @@ dependencies { implementation project(':halyard-config') implementation project(':halyard-core') + implementation project(':halyard-backup') testImplementation 'org.spockframework:spock-core:1.3-groovy-2.5' testImplementation 'org.springframework:spring-test' diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/DynamicValidationService.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/DynamicValidationService.java index ec8d81f7f8..5aef4657b9 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/DynamicValidationService.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/DynamicValidationService.java @@ -58,7 +58,14 @@ public List validate( applicationContext, halConfig.getDeploymentConfigurations().get(0), failFast); - return run.run(); + List problems = run.run(); + if (!problems.isEmpty()) { + log.info( + "Found {} problems: {}", + problems.size(), + problems.stream().map(Problem::getMessage).collect(Collectors.toList())); + } + return problems; } finally { fileRequestService.cleanup(); } diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/GenerateService.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/GenerateService.java index 3e3e642cd3..4ead205f81 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/GenerateService.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/services/v1/GenerateService.java @@ -56,7 +56,6 @@ public class GenerateService { @Autowired protected DeploymentService deploymentService; @Autowired protected ServiceProviderFactory serviceProviderFactory; - @Autowired private HalconfigDirectoryStructure halconfigDirectoryStructure; @Autowired protected ConfigParser configParser; diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/KubernetesV2ClouddriverProfileFactory.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/KubernetesV2ClouddriverProfileFactory.java index 884b89c587..0065463be7 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/KubernetesV2ClouddriverProfileFactory.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/KubernetesV2ClouddriverProfileFactory.java @@ -22,12 +22,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spinnaker.halyard.config.model.v1.node.Providers; import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesAccount; +import com.netflix.spinnaker.halyard.config.services.v1.FileService; import com.netflix.spinnaker.halyard.core.error.v1.HalException; import com.netflix.spinnaker.halyard.core.problem.v1.Problem; -import com.netflix.spinnaker.halyard.core.secrets.v1.SecretSessionManager; -import com.netflix.spinnaker.kork.configserver.CloudConfigResourceService; -import com.netflix.spinnaker.kork.configserver.ConfigFileService; -import com.netflix.spinnaker.kork.secrets.EncryptedSecret; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -42,26 +39,16 @@ @Component @Slf4j public class KubernetesV2ClouddriverProfileFactory extends ClouddriverProfileFactory { - private final ObjectMapper objectMapper; + private final ObjectMapper objectMapper; private final Yaml yamlParser; - - private final SecretSessionManager secretSessionManager; - - private final ConfigFileService configFileService; - private final CloudConfigResourceService cloudConfigResourceService; + private final FileService fileService; public KubernetesV2ClouddriverProfileFactory( - ObjectMapper objectMapper, - Yaml yamlParser, - SecretSessionManager secretSessionManager, - ConfigFileService configFileService, - CloudConfigResourceService cloudConfigResourceService) { + ObjectMapper objectMapper, Yaml yamlParser, FileService fileService) { this.objectMapper = objectMapper; this.yamlParser = yamlParser; - this.secretSessionManager = secretSessionManager; - this.configFileService = configFileService; - this.cloudConfigResourceService = cloudConfigResourceService; + this.fileService = fileService; } @Override @@ -155,17 +142,8 @@ private void processKubernetesAccount(KubernetesAccount account) { private String getKubconfigFileContents(String kubeconfigFile) { try { - if (EncryptedSecret.isEncryptedSecret(kubeconfigFile)) { - return secretSessionManager.decrypt(kubeconfigFile); - } - - String localPath = kubeconfigFile; - if (CloudConfigResourceService.isCloudConfigResource(kubeconfigFile)) { - localPath = cloudConfigResourceService.getLocalPath(kubeconfigFile); - } - - return configFileService.getContents(localPath); - } catch (Exception e) { + return fileService.getFileContents(kubeconfigFile); + } catch (IOException e) { throw new IllegalStateException( "Failed to read kubeconfig file '" + kubeconfigFile diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/ProfileFactory.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/ProfileFactory.java index 1880bd403c..817cfcd1c6 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/ProfileFactory.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/ProfileFactory.java @@ -18,6 +18,7 @@ package com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile; import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.halyard.backup.services.v1.BackupService; import com.netflix.spinnaker.halyard.config.config.v1.HalconfigDirectoryStructure; import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration; import com.netflix.spinnaker.halyard.config.model.v1.node.Node; @@ -41,6 +42,8 @@ public abstract class ProfileFactory { @Autowired protected SecretSessionManager secretSessionManager; + @Autowired private BackupService backupService; + protected String getMinimumSecretDecryptionVersion(String deploymentName) { return null; } @@ -116,8 +119,8 @@ private String getEditWarning() { * @return the list of files required by the node to function. */ protected List backupRequiredFiles(Node node, String deploymentName) { - return node.backupLocalFiles( - halconfigDirectoryStructure.getStagingDependenciesPath(deploymentName).toString()); + return backupService.backupLocalFiles( + node, halconfigDirectoryStructure.getStagingDependenciesPath(deploymentName).toString()); } protected String yamlToString(String deploymentName, Profile profile, Object o) { diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/deck/DeckProfileFactory.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/deck/DeckProfileFactory.java index e056841976..316247e852 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/deck/DeckProfileFactory.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/profile/deck/DeckProfileFactory.java @@ -32,6 +32,8 @@ import com.netflix.spinnaker.halyard.config.model.v1.providers.dcos.DCOSProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.ecs.EcsProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.google.GoogleProvider; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudAccount; +import com.netflix.spinnaker.halyard.config.model.v1.providers.huaweicloud.HuaweiCloudProvider; import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesProvider; import com.netflix.spinnaker.halyard.config.model.v1.security.UiSecurity; import com.netflix.spinnaker.halyard.config.services.v1.AccountService; @@ -202,6 +204,23 @@ protected void setProfile( deploymentConfiguration.getProviders().getCloudfoundry(); bindings.put("cloudfoundry.default.account", cloudFoundryProvider.getPrimaryAccount()); + // Configure HuaweiCloud + HuaweiCloudProvider huaweiCloudProvider = + deploymentConfiguration.getProviders().getHuaweicloud(); + bindings.put("huaweicloud.default.account", huaweiCloudProvider.getPrimaryAccount()); + if (huaweiCloudProvider.getPrimaryAccount() != null) { + HuaweiCloudAccount huaweiCloudAccount = + (HuaweiCloudAccount) + accountService.getProviderAccount( + deploymentConfiguration.getName(), + "huaweicloud", + huaweiCloudProvider.getPrimaryAccount()); + List regionList = huaweiCloudAccount.getRegions(); + if (!regionList.isEmpty()) { + bindings.put("huaweicloud.default.region", regionList.get(0)); + } + } + // Configure notifications bindings.put("notifications.enabled", notifications.isEnabled() + ""); diff --git a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/v2/KubernetesV2Utils.java b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/v2/KubernetesV2Utils.java index b6e0e25254..bd7555db7f 100644 --- a/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/v2/KubernetesV2Utils.java +++ b/halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/v2/KubernetesV2Utils.java @@ -21,16 +21,25 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spinnaker.halyard.config.model.v1.providers.kubernetes.KubernetesAccount; +import com.netflix.spinnaker.halyard.config.services.v1.FileService; +import com.netflix.spinnaker.halyard.core.error.v1.HalException; +import com.netflix.spinnaker.halyard.core.problem.v1.Problem; +import com.netflix.spinnaker.halyard.core.resource.v1.JinjaJarResource; import com.netflix.spinnaker.halyard.core.resource.v1.TemplatedResource; import com.netflix.spinnaker.halyard.core.secrets.v1.SecretSessionManager; import com.netflix.spinnaker.kork.configserver.CloudConfigResourceService; -import com.netflix.spinnaker.kork.secrets.EncryptedSecret; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.yaml.snakeyaml.Yaml; @@ -45,11 +54,15 @@ public class KubernetesV2Utils { private final CloudConfigResourceService cloudConfigResourceService; + private final FileService fileService; + public KubernetesV2Utils( SecretSessionManager secretSessionManager, - CloudConfigResourceService cloudConfigResourceService) { + CloudConfigResourceService cloudConfigResourceService, + FileService fileService) { this.secretSessionManager = secretSessionManager; this.cloudConfigResourceService = cloudConfigResourceService; + this.fileService = fileService; } public List kubectlPrefix(KubernetesAccount account) { @@ -66,29 +79,15 @@ public List kubectlPrefix(KubernetesAccount account) { command.add(context); } - String kubeconfig = getKubeconfigFile(account); - if (kubeconfig != null && !kubeconfig.isEmpty()) { + Path kubeconfig = fileService.getLocalFilePath(account.getKubeconfigFile()); + if (kubeconfig != null) { command.add("--kubeconfig"); - command.add(kubeconfig); + command.add(kubeconfig.toString()); } return command; } - private String getKubeconfigFile(KubernetesAccount account) { - String kubeconfigFile = account.getKubeconfigFile(); - - if (EncryptedSecret.isEncryptedSecret(kubeconfigFile)) { - return secretSessionManager.decryptAsFile(kubeconfigFile); - } - - if (CloudConfigResourceService.isCloudConfigResource(kubeconfigFile)) { - return cloudConfigResourceService.getLocalPath(kubeconfigFile); - } - - return kubeconfigFile; - } - List kubectlPodServiceCommand( KubernetesAccount account, String namespace, String service) { List command = kubectlPrefix(account); @@ -121,6 +120,49 @@ List kubectlConnectPodCommand( return command; } + public SecretSpec createSecretSpec( + String namespace, String clusterName, String name, List files) { + Map contentMap = new HashMap<>(); + for (SecretMountPair pair : files) { + String contents; + if (pair.getContentBytes() != null) { + contents = new String(Base64.getEncoder().encode(pair.getContentBytes())); + } else { + try { + contents = + new String( + Base64.getEncoder() + .encode(IOUtils.toByteArray(new FileInputStream(pair.getContents())))); + } catch (IOException e) { + throw new HalException( + Problem.Severity.FATAL, + "Failed to read required config file: " + + pair.getContents().getAbsolutePath() + + ": " + + e.getMessage(), + e); + } + } + + contentMap.put(pair.getName(), contents); + } + + SecretSpec spec = new SecretSpec(); + spec.name = name + "-" + Math.abs(contentMap.hashCode()); + + spec.resource = new JinjaJarResource("/kubernetes/manifests/secret.yml"); + Map bindings = new HashMap<>(); + + bindings.put("files", contentMap); + bindings.put("name", spec.name); + bindings.put("namespace", namespace); + bindings.put("clusterName", clusterName); + + spec.resource.extendBindings(bindings); + + return spec; + } + public String prettify(String input) { Yaml yaml = new Yaml(new SafeConstructor()); return yaml.dump(yaml.load(input)); diff --git a/halyard-deploy/src/main/resources/kubernetes/manifests/deployment.yml b/halyard-deploy/src/main/resources/kubernetes/manifests/deployment.yml index a1f9868590..bc7c3b1632 100644 --- a/halyard-deploy/src/main/resources/kubernetes/manifests/deployment.yml +++ b/halyard-deploy/src/main/resources/kubernetes/manifests/deployment.yml @@ -31,10 +31,10 @@ spec: {% endif %} template: metadata: - annotations: { - {% for key, value in podAnnotations.items() %} - "{{ key }}" : "{{ value }}"{% if not loop.last %}, {% endif %} - {% endfor %}} + annotations: + {% for key, value in podAnnotations.items() %} + "{{ key }}": "{{ value }}" + {% endfor %} labels: app: spin cluster: spin-{{ name }} @@ -43,6 +43,6 @@ spec: app.kubernetes.io/part-of: spinnaker app.kubernetes.io/version: {{ version }} {% for key, value in podLabels.items() %} - "{{ key }}" : "{{ value }}"{% if not loop.last %}, {% endif %} + "{{ key }}": "{{ value }}" {% endfor %} spec: {{ podSpec }} diff --git a/halyard-deploy/src/main/resources/kubernetes/manifests/service.yml b/halyard-deploy/src/main/resources/kubernetes/manifests/service.yml index 90bda8fac6..590f084541 100644 --- a/halyard-deploy/src/main/resources/kubernetes/manifests/service.yml +++ b/halyard-deploy/src/main/resources/kubernetes/manifests/service.yml @@ -7,7 +7,7 @@ metadata: app: spin cluster: spin-{{ name }} {% for key, value in serviceLabels.items() %} - "{{ key }}" : "{{ value }}"{% if not loop.last %}, {% endif %} + "{{ key }}": "{{ value }}" {% endfor %} spec: selector: