diff --git a/docs/README.md b/docs/README.md index 4412c76894..5259d3849b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -46,6 +46,7 @@ * [`kubernetes`](./module-types/kubernetes.md) * [`maven-container`](./module-types/maven-container.md) * [`openfaas`](./module-types/openfaas.md) + * [`persistentvolumeclaim`](./module-types/persistentvolumeclaim.md) * [`terraform`](./module-types/terraform.md) * [Reference](./reference/README.md) * [Glossary](./reference/glossary.md) diff --git a/docs/guides/container-modules.md b/docs/guides/container-modules.md index 777be1668b..c8638e84c3 100644 --- a/docs/guides/container-modules.md +++ b/docs/guides/container-modules.md @@ -216,3 +216,92 @@ values: Here, we declare `my-image` as a dependency for the `my-service` Helm chart. In order for the Helm chart to be able to reference the built container image, we must provide the correct image name and version. For a full list of keys that are available for the `container` module type, take a look at the [outputs reference](../module-types/container.md#outputs). + +## Mounting volumes + +`container` services, tasks and tests can all mount volumes, using _volume modules_. One such is the [`persistentvolumeclaim` module type](../module-types/persistentvolumeclaim.md), supported by the `kubernetes` provider. To mount a volume, you need to define a volume module, and reference it using the `volumes` key on your services, tasks and/or tests. + +Example: + +```yaml +kind: Module +name: my-volume +type: persistentvolumeclaim +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 1Gi +--- +kind: Module +name: my-module +type: container +services: + - name: my-service + replicas: 1 # <- Important! Unless your volume supports ReadWriteMany, you can't run multiple replicas with it + volumes: + - name: my-volume + module: my-volume + containerPath: /volume + ... +``` + +This will mount the `my-volume` PVC at `/volume` in the `my-service` service when it is run. The `my-volume` module creates a `PersistentVolumeClaim` resource in your project namespace, and the `spec` field is passed directly to the same field on the PVC resource. + +{% hint style="warning" %} +Notice the `accessModes` field in the volume module above. The default storage classes in Kubernetes generally don't support being mounted by multiple Pods at the same time. If your volume module doesn't support the `ReadWriteMany` access mode, you must take care not to use the same volume in multiple services, tasks or tests, or multiple replicas. See [Shared volumes](#shared-volumes) below for how to share a single volume with multiple Pods. +{% endhint %} + +You can do the same for tests and tasks using the [`tests.volumes`](../module-types/container.md#testsvolumes) and [`tasks.volumes`](../module-types/container.md#tasksvolumes) fields. `persistentvolumeclaim` volumes can of course also be referenced in `kubernetes` and +`helm` modules, since they are deployed as standard PersistentVolumeClaim resources. + +Take a look at the [`persistentvolumeclaim` module type](../module-types/persistentvolumeclaim.md) and [`container` module](../module-types/container.md#servicesvolumes) docs for more details. + +### Shared volumes + +For a volume to be shared between multiple replicas, or multiple services, tasks and/or tests, it needs to be configured with a storage class (using the `storageClassName` field) that supports the `ReadWriteMany` (RWX) access mode. The available storage classes that support RWX vary by cloud providers and cluster setups, and in many cases you need to define a `StorageClass` or deploy a _storage class provisioner_ to your cluster. + +You can find a list of storage options and their supported access modes [here](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). Here are a few commonly used RWX provisioners and storage classes: + +* [NFS Server Provisioner](https://github.com/helm/charts/tree/master/stable/nfs-server-provisioner) +* [Azure File](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv) +* [AWS EFS Provisioner](https://github.com/helm/charts/tree/master/stable/efs-provisioner) +* [Ceph (via Rook)](https://rook.io/docs/rook/v1.2/ceph-filesystem.html) + +Once any of those is set up you can create a `persistentvolumeclaim` module that uses the configured storage class. Here, for example, is how you might use a shared volume with a configured `azurefile` storage class: + +```yaml +kind: Module +name: shared-volume +type: persistentvolumeclaim +spec: + accessModes: [ReadWriteMany] + resources: + requests: + storage: 1Gi + storageClassName: azurefile +--- +kind: Module +name: my-module +type: container +services: + - name: my-service + volumes: + - &volume # <- using a YAML anchor to re-use the volume spec in tasks and tests + name: shared-volume + module: shared-volume + containerPath: /volume + ... +tasks: + - name: my-task + volumes: + - *volume + ... +tests: + - name: my-test + volumes: + - *volume + ... +``` + +Here the same volume is used across a service, task and a test in the same module. You could similarly use the same volume across multiple container modules. diff --git a/docs/module-types/README.md b/docs/module-types/README.md index 472bb1a03b..726a42e898 100644 --- a/docs/module-types/README.md +++ b/docs/module-types/README.md @@ -13,4 +13,5 @@ title: Module Types * [`kubernetes`](./kubernetes.md) * [`maven-container`](./maven-container.md) * [`openfaas`](./openfaas.md) +* [`persistentvolumeclaim`](./persistentvolumeclaim.md) * [`terraform`](./terraform.md) \ No newline at end of file diff --git a/docs/module-types/container.md b/docs/module-types/container.md index 2369b4cebe..441d45f586 100644 --- a/docs/module-types/container.md +++ b/docs/module-types/container.md @@ -292,7 +292,10 @@ services: # with hot-reloading enabled, or if the provider doesn't support multiple replicas. replicas: - # List of volumes that should be mounted when deploying the container. + # List of volumes that should be mounted when deploying the service. + # + # Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when + # deploying the container. volumes: - # The name of the allocated volume. name: @@ -307,6 +310,18 @@ services: # module source path (or absolute). hostPath: + # The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will + # depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim + # module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + # + # When a `module` is specified, the referenced module/volume will be automatically configured as a runtime + # dependency of this service, as well as a build dependency of this module. + # + # Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports + # the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple + # services at the same time. Refer to the documentation of the module type in question to learn more. + module: + # A list of tests to run in the module. tests: - # The name of the test. @@ -345,6 +360,36 @@ tests: # `GARDEN`) and values must be primitives or references to secrets. env: {} + # List of volumes that should be mounted when deploying the test. + # + # Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when + # deploying the container. + volumes: + - # The name of the allocated volume. + name: + + # The path where the volume should be mounted in the container. + containerPath: + + # _NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms + # and providers. Some providers may not support it at all._ + # + # A local path or path on the node that's running the container, to mount in the container, relative to the + # module source path (or absolute). + hostPath: + + # The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will + # depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim + # module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + # + # When a `module` is specified, the referenced module/volume will be automatically configured as a runtime + # dependency of this service, as well as a build dependency of this module. + # + # Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports + # the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple + # services at the same time. Refer to the documentation of the module type in question to learn more. + module: + # A list of tasks that can be run from this container module. These can be used as dependencies for services (executed # before the service is deployed) or for other tasks. tasks: @@ -398,6 +443,36 @@ tasks: # Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with # `GARDEN`) and values must be primitives or references to secrets. env: {} + + # List of volumes that should be mounted when deploying the task. + # + # Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when + # deploying the container. + volumes: + - # The name of the allocated volume. + name: + + # The path where the volume should be mounted in the container. + containerPath: + + # _NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms + # and providers. Some providers may not support it at all._ + # + # A local path or path on the node that's running the container, to mount in the container, relative to the + # module source path (or absolute). + hostPath: + + # The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will + # depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim + # module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + # + # When a `module` is specified, the referenced module/volume will be automatically configured as a runtime + # dependency of this service, as well as a build dependency of this module. + # + # Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports + # the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple + # services at the same time. Refer to the documentation of the module type in question to learn more. + module: ``` ## Configuration Keys @@ -1191,7 +1266,9 @@ Note: This setting may be overridden or ignored in some cases. For example, when [services](#services) > volumes -List of volumes that should be mounted when deploying the container. +List of volumes that should be mounted when deploying the service. + +Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when deploying the container. | Type | Default | Required | | --------------- | ------- | -------- | @@ -1239,6 +1316,20 @@ services: - hostPath: "/some/dir" ``` +### `services[].volumes[].module` + +[services](#services) > [volumes](#servicesvolumes) > module + +The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + +When a `module` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + +Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `tests[]` A list of tests to run in the module. @@ -1408,6 +1499,74 @@ tests: - {} ``` +### `tests[].volumes[]` + +[tests](#tests) > volumes + +List of volumes that should be mounted when deploying the test. + +Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when deploying the container. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `tests[].volumes[].name` + +[tests](#tests) > [volumes](#testsvolumes) > name + +The name of the allocated volume. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tests[].volumes[].containerPath` + +[tests](#tests) > [volumes](#testsvolumes) > containerPath + +The path where the volume should be mounted in the container. + +| Type | Required | +| ----------- | -------- | +| `posixPath` | Yes | + +### `tests[].volumes[].hostPath` + +[tests](#tests) > [volumes](#testsvolumes) > hostPath + +_NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms +and providers. Some providers may not support it at all._ + +A local path or path on the node that's running the container, to mount in the container, relative to the +module source path (or absolute). + +| Type | Required | +| ----------- | -------- | +| `posixPath` | No | + +Example: + +```yaml +tests: + - volumes: + - hostPath: "/some/dir" +``` + +### `tests[].volumes[].module` + +[tests](#tests) > [volumes](#testsvolumes) > module + +The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + +When a `module` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + +Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `tasks[]` A list of tasks that can be run from this container module. These can be used as dependencies for services (executed before the service is deployed) or for other tasks. @@ -1598,6 +1757,74 @@ tasks: - {} ``` +### `tasks[].volumes[]` + +[tasks](#tasks) > volumes + +List of volumes that should be mounted when deploying the task. + +Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when deploying the container. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `tasks[].volumes[].name` + +[tasks](#tasks) > [volumes](#tasksvolumes) > name + +The name of the allocated volume. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tasks[].volumes[].containerPath` + +[tasks](#tasks) > [volumes](#tasksvolumes) > containerPath + +The path where the volume should be mounted in the container. + +| Type | Required | +| ----------- | -------- | +| `posixPath` | Yes | + +### `tasks[].volumes[].hostPath` + +[tasks](#tasks) > [volumes](#tasksvolumes) > hostPath + +_NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms +and providers. Some providers may not support it at all._ + +A local path or path on the node that's running the container, to mount in the container, relative to the +module source path (or absolute). + +| Type | Required | +| ----------- | -------- | +| `posixPath` | No | + +Example: + +```yaml +tasks: + - volumes: + - hostPath: "/some/dir" +``` + +### `tasks[].volumes[].module` + +[tasks](#tasks) > [volumes](#tasksvolumes) > module + +The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + +When a `module` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + +Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ## Outputs diff --git a/docs/module-types/maven-container.md b/docs/module-types/maven-container.md index 048646aa0a..5ec3f93626 100644 --- a/docs/module-types/maven-container.md +++ b/docs/module-types/maven-container.md @@ -290,7 +290,10 @@ services: # with hot-reloading enabled, or if the provider doesn't support multiple replicas. replicas: - # List of volumes that should be mounted when deploying the container. + # List of volumes that should be mounted when deploying the service. + # + # Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when + # deploying the container. volumes: - # The name of the allocated volume. name: @@ -305,6 +308,18 @@ services: # module source path (or absolute). hostPath: + # The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will + # depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim + # module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + # + # When a `module` is specified, the referenced module/volume will be automatically configured as a runtime + # dependency of this service, as well as a build dependency of this module. + # + # Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports + # the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple + # services at the same time. Refer to the documentation of the module type in question to learn more. + module: + # A list of tests to run in the module. tests: - # The name of the test. @@ -343,6 +358,36 @@ tests: # `GARDEN`) and values must be primitives or references to secrets. env: {} + # List of volumes that should be mounted when deploying the test. + # + # Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when + # deploying the container. + volumes: + - # The name of the allocated volume. + name: + + # The path where the volume should be mounted in the container. + containerPath: + + # _NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms + # and providers. Some providers may not support it at all._ + # + # A local path or path on the node that's running the container, to mount in the container, relative to the + # module source path (or absolute). + hostPath: + + # The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will + # depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim + # module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + # + # When a `module` is specified, the referenced module/volume will be automatically configured as a runtime + # dependency of this service, as well as a build dependency of this module. + # + # Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports + # the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple + # services at the same time. Refer to the documentation of the module type in question to learn more. + module: + # A list of tasks that can be run from this container module. These can be used as dependencies for services (executed # before the service is deployed) or for other tasks. tasks: @@ -397,6 +442,36 @@ tasks: # `GARDEN`) and values must be primitives or references to secrets. env: {} + # List of volumes that should be mounted when deploying the task. + # + # Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when + # deploying the container. + volumes: + - # The name of the allocated volume. + name: + + # The path where the volume should be mounted in the container. + containerPath: + + # _NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms + # and providers. Some providers may not support it at all._ + # + # A local path or path on the node that's running the container, to mount in the container, relative to the + # module source path (or absolute). + hostPath: + + # The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will + # depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim + # module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + # + # When a `module` is specified, the referenced module/volume will be automatically configured as a runtime + # dependency of this service, as well as a build dependency of this module. + # + # Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports + # the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple + # services at the same time. Refer to the documentation of the module type in question to learn more. + module: + # Set this to override the default OpenJDK container image version. Make sure the image version matches the # configured `jdkVersion`. Ignored if you provide your own Dockerfile. imageVersion: @@ -1199,7 +1274,9 @@ Note: This setting may be overridden or ignored in some cases. For example, when [services](#services) > volumes -List of volumes that should be mounted when deploying the container. +List of volumes that should be mounted when deploying the service. + +Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when deploying the container. | Type | Default | Required | | --------------- | ------- | -------- | @@ -1247,6 +1324,20 @@ services: - hostPath: "/some/dir" ``` +### `services[].volumes[].module` + +[services](#services) > [volumes](#servicesvolumes) > module + +The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + +When a `module` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + +Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `tests[]` A list of tests to run in the module. @@ -1416,6 +1507,74 @@ tests: - {} ``` +### `tests[].volumes[]` + +[tests](#tests) > volumes + +List of volumes that should be mounted when deploying the test. + +Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when deploying the container. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `tests[].volumes[].name` + +[tests](#tests) > [volumes](#testsvolumes) > name + +The name of the allocated volume. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tests[].volumes[].containerPath` + +[tests](#tests) > [volumes](#testsvolumes) > containerPath + +The path where the volume should be mounted in the container. + +| Type | Required | +| ----------- | -------- | +| `posixPath` | Yes | + +### `tests[].volumes[].hostPath` + +[tests](#tests) > [volumes](#testsvolumes) > hostPath + +_NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms +and providers. Some providers may not support it at all._ + +A local path or path on the node that's running the container, to mount in the container, relative to the +module source path (or absolute). + +| Type | Required | +| ----------- | -------- | +| `posixPath` | No | + +Example: + +```yaml +tests: + - volumes: + - hostPath: "/some/dir" +``` + +### `tests[].volumes[].module` + +[tests](#tests) > [volumes](#testsvolumes) > module + +The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + +When a `module` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + +Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `tasks[]` A list of tasks that can be run from this container module. These can be used as dependencies for services (executed before the service is deployed) or for other tasks. @@ -1606,6 +1765,74 @@ tasks: - {} ``` +### `tasks[].volumes[]` + +[tasks](#tasks) > volumes + +List of volumes that should be mounted when deploying the task. + +Note: If neither `hostPath` nor `module` is specified, an empty ephemeral volume is created and mounted when deploying the container. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `tasks[].volumes[].name` + +[tasks](#tasks) > [volumes](#tasksvolumes) > name + +The name of the allocated volume. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tasks[].volumes[].containerPath` + +[tasks](#tasks) > [volumes](#tasksvolumes) > containerPath + +The path where the volume should be mounted in the container. + +| Type | Required | +| ----------- | -------- | +| `posixPath` | Yes | + +### `tasks[].volumes[].hostPath` + +[tasks](#tasks) > [volumes](#tasksvolumes) > hostPath + +_NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms +and providers. Some providers may not support it at all._ + +A local path or path on the node that's running the container, to mount in the container, relative to the +module source path (or absolute). + +| Type | Required | +| ----------- | -------- | +| `posixPath` | No | + +Example: + +```yaml +tasks: + - volumes: + - hostPath: "/some/dir" +``` + +### `tasks[].volumes[].module` + +[tasks](#tasks) > [volumes](#tasksvolumes) > module + +The name of a _volume module_ that should be mounted at `containerPath`. The supported module types will depend on which provider you are using. The `kubernetes` provider supports the [persistentvolumeclaim module](https://docs.garden.io/module-types/persistentvolumeclaim), for example. + +When a `module` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + +Note: Make sure to pay attention to the supported `accessModes` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + +| Type | Required | +| -------- | -------- | +| `string` | No | + ### `imageVersion` Set this to override the default OpenJDK container image version. Make sure the image version matches the diff --git a/docs/module-types/persistentvolumeclaim.md b/docs/module-types/persistentvolumeclaim.md new file mode 100644 index 0000000000..1ae120caa0 --- /dev/null +++ b/docs/module-types/persistentvolumeclaim.md @@ -0,0 +1,577 @@ +--- +title: "`persistentvolumeclaim` Module Type" +tocTitle: "`persistentvolumeclaim`" +--- + +# `persistentvolumeclaim` Module Type + +## Description + +Creates a [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) in your namespace, that can be referenced and mounted by other resources and [container modules](https://docs.garden.io/module-types/container). + +See the [Mounting volumes](https://docs.garden.io/guides/container-modules#mounting-volumes) guide for more info and usage examples. + +Below is the full schema reference. For an introduction to configuring Garden modules, please look at our [Configuration +guide](../guides/configuration-files.md). + +The [first section](#complete-yaml-schema) contains the complete YAML schema, and the [second section](#configuration-keys) describes each schema key. + +`persistentvolumeclaim` modules also export values that are available in template strings. See the [Outputs](#outputs) section below for details. + +## Complete YAML Schema + +The values in the schema below are the default values. + +```yaml +# The schema version of this module's config (currently not used). +apiVersion: garden.io/v0 + +kind: Module + +# The type of this module. +type: + +# The name of this module. +name: + +# A description of the module. +description: + +# Set this to `true` to disable the module. You can use this with conditional template strings to disable modules +# based on, for example, the current environment or other variables (e.g. `disabled: \${environment.name == "prod"}`). +# This can be handy when you only need certain modules for specific environments, e.g. only for development. +# +# Disabling a module means that any services, tasks and tests contained in it will not be deployed or run. It also +# means that the module is not built _unless_ it is declared as a build dependency by another enabled module (in which +# case building this module is necessary for the dependant to be built). +# +# If you disable the module, and its services, tasks or tests are referenced as _runtime_ dependencies, Garden will +# automatically ignore those dependency declarations. Note however that template strings referencing the module's +# service or task outputs (i.e. runtime outputs) will fail to resolve when the module is disabled, so you need to make +# sure to provide alternate values for those if you're using them, using conditional expressions. +disabled: false + +# Specify a list of POSIX-style paths or globs that should be regarded as the source files for this module. Files that +# do *not* match these paths or globs are excluded when computing the version of the module, when responding to +# filesystem watch events, and when staging builds. +# +# Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your source +# tree, which use the same format as `.gitignore` files. See the [Configuration Files +# guide](https://docs.garden.io/guides/configuration-files#including-excluding-files-and-directories) for details. +# +# Also note that specifying an empty list here means _no sources_ should be included. +include: + +# Specify a list of POSIX-style paths or glob patterns that should be excluded from the module. Files that match these +# paths or globs are excluded when computing the version of the module, when responding to filesystem watch events, +# and when staging builds. +# +# Note that you can also explicitly _include_ files using the `include` field. If you also specify the `include` +# field, the files/patterns specified here are filtered from the files matched by `include`. See the [Configuration +# Files guide](https://docs.garden.io/guides/configuration-files#including-excluding-files-and-directories)for +# details. +# +# Unlike the `modules.exclude` field in the project config, the filters here have _no effect_ on which files and +# directories are watched for changes. Use the project `modules.exclude` field to affect those, if you have large +# directories that should not be watched for changes. +exclude: + +# A remote repository URL. Currently only supports git servers. Must contain a hash suffix pointing to a specific +# branch or tag, with the format: # +# +# Garden will import the repository source code into this module, but read the module's config from the local +# garden.yml file. +repositoryUrl: + +# When false, disables pushing this module to remote registries. +allowPublish: true + +# Specify how to build the module. Note that plugins may define additional keys on this object. +build: + # A list of modules that must be built before this module is built. + dependencies: + - # Module name to build ahead of this module. + name: + + # Specify one or more files or directories to copy from the built dependency to this module. + copy: + - # POSIX-style path or filename of the directory or file(s) to copy to the target. + source: + + # POSIX-style path or filename to copy the directory or file(s), relative to the build directory. + # Defaults to to same as source path. + target: '' + +# List of services and tasks to deploy/run before deploying this PVC. +dependencies: + +# The namespace to deploy the PVC in. Note that any module referencing the PVC must be in the same namespace, so in +# most cases you should leave this unset. +namespace: + +# The spec for the PVC. This is passed directly to the created PersistentVolumeClaim resource. Note that the spec +# schema may include (or even require) additional fields, depending on the used `storageClass`. See the +# [PersistentVolumeClaim docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) +# for details. +spec: + # AccessModes contains the desired access modes the volume should have. More info: + # https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + accessModes: + + # TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the + # same namespace. + dataSource: + # APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must + # be in the core API group. For any other third-party types, APIGroup is required. + apiGroup: + + # Kind is the type of resource being referenced + kind: + + # Name is the name of resource being referenced + name: + + # ResourceRequirements describes the compute resource requirements. + resources: + # Limits describes the maximum amount of compute resources allowed. More info: + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + limits: + + # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it + # defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + requests: + + # A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are + # ANDed. An empty label selector matches all objects. A null label selector matches no objects. + selector: + # matchExpressions is a list of label selector requirements. The requirements are ANDed. + matchExpressions: + + # matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an + # element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only + # "value". The requirements are ANDed. + matchLabels: + + # Name of the StorageClass required by the claim. More info: + # https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + storageClassName: + + # volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included + # in claim spec. This is a beta feature. + volumeMode: + + # VolumeName is the binding reference to the PersistentVolume backing this claim. + volumeName: +``` + +## Configuration Keys + +### `apiVersion` + +The schema version of this module's config (currently not used). + +| Type | Allowed Values | Default | Required | +| -------- | -------------- | ---------------- | -------- | +| `string` | "garden.io/v0" | `"garden.io/v0"` | Yes | + +### `kind` + +| Type | Allowed Values | Default | Required | +| -------- | -------------- | ---------- | -------- | +| `string` | "Module" | `"Module"` | Yes | + +### `type` + +The type of this module. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +Example: + +```yaml +type: "container" +``` + +### `name` + +The name of this module. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +Example: + +```yaml +name: "my-sweet-module" +``` + +### `description` + +A description of the module. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `disabled` + +Set this to `true` to disable the module. You can use this with conditional template strings to disable modules based on, for example, the current environment or other variables (e.g. `disabled: \${environment.name == "prod"}`). This can be handy when you only need certain modules for specific environments, e.g. only for development. + +Disabling a module means that any services, tasks and tests contained in it will not be deployed or run. It also means that the module is not built _unless_ it is declared as a build dependency by another enabled module (in which case building this module is necessary for the dependant to be built). + +If you disable the module, and its services, tasks or tests are referenced as _runtime_ dependencies, Garden will automatically ignore those dependency declarations. Note however that template strings referencing the module's service or task outputs (i.e. runtime outputs) will fail to resolve when the module is disabled, so you need to make sure to provide alternate values for those if you're using them, using conditional expressions. + +| Type | Default | Required | +| --------- | ------- | -------- | +| `boolean` | `false` | No | + +### `include[]` + +Specify a list of POSIX-style paths or globs that should be regarded as the source files for this module. Files that do *not* match these paths or globs are excluded when computing the version of the module, when responding to filesystem watch events, and when staging builds. + +Note that you can also _exclude_ files using the `exclude` field or by placing `.gardenignore` files in your source tree, which use the same format as `.gitignore` files. See the [Configuration Files guide](https://docs.garden.io/guides/configuration-files#including-excluding-files-and-directories) for details. + +Also note that specifying an empty list here means _no sources_ should be included. + +| Type | Required | +| ------------------ | -------- | +| `array[posixPath]` | No | + +Example: + +```yaml +include: + - Dockerfile + - my-app.js +``` + +### `exclude[]` + +Specify a list of POSIX-style paths or glob patterns that should be excluded from the module. Files that match these paths or globs are excluded when computing the version of the module, when responding to filesystem watch events, and when staging builds. + +Note that you can also explicitly _include_ files using the `include` field. If you also specify the `include` field, the files/patterns specified here are filtered from the files matched by `include`. See the [Configuration Files guide](https://docs.garden.io/guides/configuration-files#including-excluding-files-and-directories)for details. + +Unlike the `modules.exclude` field in the project config, the filters here have _no effect_ on which files and directories are watched for changes. Use the project `modules.exclude` field to affect those, if you have large directories that should not be watched for changes. + +| Type | Required | +| ------------------ | -------- | +| `array[posixPath]` | No | + +Example: + +```yaml +exclude: + - tmp/**/* + - '*.log' +``` + +### `repositoryUrl` + +A remote repository URL. Currently only supports git servers. Must contain a hash suffix pointing to a specific branch or tag, with the format: # + +Garden will import the repository source code into this module, but read the module's config from the local garden.yml file. + +| Type | Required | +| ----------------- | -------- | +| `gitUrl | string` | No | + +Example: + +```yaml +repositoryUrl: "git+https://github.com/org/repo.git#v2.0" +``` + +### `allowPublish` + +When false, disables pushing this module to remote registries. + +| Type | Default | Required | +| --------- | ------- | -------- | +| `boolean` | `true` | No | + +### `build` + +Specify how to build the module. Note that plugins may define additional keys on this object. + +| Type | Default | Required | +| -------- | --------------------- | -------- | +| `object` | `{"dependencies":[]}` | No | + +### `build.dependencies[]` + +[build](#build) > dependencies + +A list of modules that must be built before this module is built. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +Example: + +```yaml +build: + ... + dependencies: + - name: some-other-module-name +``` + +### `build.dependencies[].name` + +[build](#build) > [dependencies](#builddependencies) > name + +Module name to build ahead of this module. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `build.dependencies[].copy[]` + +[build](#build) > [dependencies](#builddependencies) > copy + +Specify one or more files or directories to copy from the built dependency to this module. + +| Type | Default | Required | +| --------------- | ------- | -------- | +| `array[object]` | `[]` | No | + +### `build.dependencies[].copy[].source` + +[build](#build) > [dependencies](#builddependencies) > [copy](#builddependenciescopy) > source + +POSIX-style path or filename of the directory or file(s) to copy to the target. + +| Type | Required | +| ----------- | -------- | +| `posixPath` | Yes | + +### `build.dependencies[].copy[].target` + +[build](#build) > [dependencies](#builddependencies) > [copy](#builddependenciescopy) > target + +POSIX-style path or filename to copy the directory or file(s), relative to the build directory. +Defaults to to same as source path. + +| Type | Default | Required | +| ----------- | ------- | -------- | +| `posixPath` | `""` | No | + +### `dependencies` + +List of services and tasks to deploy/run before deploying this PVC. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `namespace` + +The namespace to deploy the PVC in. Note that any module referencing the PVC must be in the same namespace, so in most cases you should leave this unset. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `spec` + +The spec for the PVC. This is passed directly to the created PersistentVolumeClaim resource. Note that the spec schema may include (or even require) additional fields, depending on the used `storageClass`. See the [PersistentVolumeClaim docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) for details. + +| Type | Required | +| -------------- | -------- | +| `customObject` | Yes | + +### `spec.accessModes[]` + +[spec](#spec) > accessModes + +AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + +| Type | Required | +| ------- | -------- | +| `array` | No | + +### `spec.dataSource` + +[spec](#spec) > dataSource + +TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `spec.dataSource.apiGroup` + +[spec](#spec) > [dataSource](#specdatasource) > apiGroup + +APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `spec.dataSource.kind` + +[spec](#spec) > [dataSource](#specdatasource) > kind + +Kind is the type of resource being referenced + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `spec.dataSource.name` + +[spec](#spec) > [dataSource](#specdatasource) > name + +Name is the name of resource being referenced + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `spec.resources` + +[spec](#spec) > resources + +ResourceRequirements describes the compute resource requirements. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `spec.resources.limits` + +[spec](#spec) > [resources](#specresources) > limits + +Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `spec.resources.requests` + +[spec](#spec) > [resources](#specresources) > requests + +Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `spec.selector` + +[spec](#spec) > selector + +A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `spec.selector.matchExpressions[]` + +[spec](#spec) > [selector](#specselector) > matchExpressions + +matchExpressions is a list of label selector requirements. The requirements are ANDed. + +| Type | Required | +| ------- | -------- | +| `array` | No | + +### `spec.selector.matchLabels` + +[spec](#spec) > [selector](#specselector) > matchLabels + +matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + +| Type | Required | +| -------- | -------- | +| `object` | No | + +### `spec.storageClassName` + +[spec](#spec) > storageClassName + +Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `spec.volumeMode` + +[spec](#spec) > volumeMode + +volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature. + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `spec.volumeName` + +[spec](#spec) > volumeName + +VolumeName is the binding reference to the PersistentVolume backing this claim. + +| Type | Required | +| -------- | -------- | +| `string` | No | + + +## Outputs + +### Module Outputs + +The following keys are available via the `${modules.}` template string key for `persistentvolumeclaim` +modules. + +### `${modules..buildPath}` + +The build path of the module. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${modules.my-module.buildPath} +``` + +### `${modules..path}` + +The local path of the module. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${modules.my-module.path} +``` + +### `${modules..version}` + +The current version of the module. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${modules.my-module.version} +``` + diff --git a/docs/module-types/terraform.md b/docs/module-types/terraform.md index 539b2cb802..bc8d14c5f4 100644 --- a/docs/module-types/terraform.md +++ b/docs/module-types/terraform.md @@ -439,9 +439,3 @@ A map of all the outputs defined in the Terraform stack. | -------- | | `object` | -### `${runtime.services..outputs.}` - -| Type | -| ----- | -| `any` | - diff --git a/docs/providers/kubernetes.md b/docs/providers/kubernetes.md index 0c4d1f53eb..6b98e81b7a 100644 --- a/docs/providers/kubernetes.md +++ b/docs/providers/kubernetes.md @@ -413,6 +413,7 @@ Set a default username (used for namespacing within a cluster). ### `providers[].deploymentStrategy` [providers](#providers) > deploymentStrategy +> ⚠️ **Experimental**: this is an experimental feature and the API might change in the future. Defines the strategy for deploying the project services. Default is "rolling update" and there is experimental support for "blue/green" deployment. diff --git a/docs/providers/local-kubernetes.md b/docs/providers/local-kubernetes.md index 52f2c5a12d..4e23d7852f 100644 --- a/docs/providers/local-kubernetes.md +++ b/docs/providers/local-kubernetes.md @@ -383,6 +383,7 @@ Set a default username (used for namespacing within a cluster). ### `providers[].deploymentStrategy` [providers](#providers) > deploymentStrategy +> ⚠️ **Experimental**: this is an experimental feature and the API might change in the future. Defines the strategy for deploying the project services. Default is "rolling update" and there is experimental support for "blue/green" deployment. diff --git a/docs/reference/template-strings.md b/docs/reference/template-strings.md index 32f0c50ecc..48a20e3e1e 100644 --- a/docs/reference/template-strings.md +++ b/docs/reference/template-strings.md @@ -43,14 +43,6 @@ A map of all local environment variables (see https://nodejs.org/api/process.htm | -------- | | `object` | -### `${local.env.}` - -The environment variable value. - -| Type | -| -------- | -| `string` | - ### `${local.platform}` A string indicating the platform that the framework is running on (see https://nodejs.org/api/process.html#process_process_platform) @@ -117,14 +109,6 @@ A map of all local environment variables (see https://nodejs.org/api/process.htm | -------- | | `object` | -### `${local.env.}` - -The environment variable value. - -| Type | -| -------- | -| `string` | - ### `${local.platform}` A string indicating the platform that the framework is running on (see https://nodejs.org/api/process.html#process_process_platform) @@ -205,38 +189,6 @@ Retrieve information about providers that are defined in the project. | -------- | ------- | | `object` | `{}` | -### `${providers..config.*}` - -The resolved configuration for the provider. - -| Type | -| -------- | -| `object` | - -### `${providers..config.}` - -The provider config key value. Refer to individual [provider references](https://docs.garden.io/providers) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - -### `${providers..outputs.*}` - -The outputs defined by the provider (see individual plugin docs for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${providers..outputs.}` - -The provider output value. Refer to individual [provider references](https://docs.garden.io/providers) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${variables.*}` A map of all variables defined in the project configuration. @@ -245,14 +197,6 @@ A map of all variables defined in the project configuration. | -------- | ------- | | `object` | `{}` | -### `${variables.}` - -The value of the variable. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${var.*}` Alias for the variables field. @@ -261,14 +205,6 @@ Alias for the variables field. | -------- | ------- | | `object` | `{}` | -### `${var.}` - -Number, string or boolean - -| Type | -| --------------------------- | -| `number | string | boolean` | - ## Module configuration context @@ -310,14 +246,6 @@ A map of all local environment variables (see https://nodejs.org/api/process.htm | -------- | | `object` | -### `${local.env.}` - -The environment variable value. - -| Type | -| -------- | -| `string` | - ### `${local.platform}` A string indicating the platform that the framework is running on (see https://nodejs.org/api/process.html#process_process_platform) @@ -398,38 +326,6 @@ Retrieve information about providers that are defined in the project. | -------- | ------- | | `object` | `{}` | -### `${providers..config.*}` - -The resolved configuration for the provider. - -| Type | -| -------- | -| `object` | - -### `${providers..config.}` - -The provider config key value. Refer to individual [provider references](https://docs.garden.io/providers) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - -### `${providers..outputs.*}` - -The outputs defined by the provider (see individual plugin docs for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${providers..outputs.}` - -The provider output value. Refer to individual [provider references](https://docs.garden.io/providers) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${variables.*}` A map of all variables defined in the project configuration. @@ -438,14 +334,6 @@ A map of all variables defined in the project configuration. | -------- | ------- | | `object` | `{}` | -### `${variables.}` - -The value of the variable. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${var.*}` Alias for the variables field. @@ -454,14 +342,6 @@ Alias for the variables field. | -------- | ------- | | `object` | `{}` | -### `${var.}` - -Number, string or boolean - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${modules.*}` Retrieve information about modules that are defined in the project. @@ -470,64 +350,6 @@ Retrieve information about modules that are defined in the project. | -------- | ------- | | `object` | `{}` | -### `${modules..buildPath}` - -The build path of the module. - -| Type | -| -------- | -| `string` | - -Example: - -```yaml -my-variable: ${modules..buildPath} -``` - -### `${modules..outputs.*}` - -The outputs defined by the module (see individual module type [references](https://docs.garden.io/module-types) for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${modules..outputs.}` - -The module output value. Refer to individual [module type references](https://docs.garden.io/module-types) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - -### `${modules..path}` - -The local path of the module. - -| Type | -| -------- | -| `string` | - -Example: - -```yaml -my-variable: ${modules..path} -``` - -### `${modules..version}` - -The current version of the module. - -| Type | -| -------- | -| `string` | - -Example: - -```yaml -my-variable: ${modules..version} -``` - ### `${runtime.*}` Runtime outputs and information from services and tasks (only resolved at runtime when deploying services and running tasks). @@ -544,22 +366,6 @@ Runtime information from the services that the service/task being run depends on | -------- | ------- | | `object` | `{}` | -### `${runtime.services..outputs.*}` - -The runtime outputs defined by the service (see individual module type [references](https://docs.garden.io/module-types) for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${runtime.services..outputs.}` - -The service output value. Refer to individual [module type references](https://docs.garden.io/module-types) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${runtime.tasks.*}` Runtime information from the tasks that the service/task being run depends on. @@ -568,22 +374,6 @@ Runtime information from the tasks that the service/task being run depends on. | -------- | ------- | | `object` | `{}` | -### `${runtime.tasks..outputs.*}` - -The runtime outputs defined by the task (see individual module type [references](https://docs.garden.io/module-types) for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${runtime.tasks..outputs.}` - -The task output value. Refer to individual [module type references](https://docs.garden.io/module-types) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ## Output configuration context @@ -625,14 +415,6 @@ A map of all local environment variables (see https://nodejs.org/api/process.htm | -------- | | `object` | -### `${local.env.}` - -The environment variable value. - -| Type | -| -------- | -| `string` | - ### `${local.platform}` A string indicating the platform that the framework is running on (see https://nodejs.org/api/process.html#process_process_platform) @@ -713,38 +495,6 @@ Retrieve information about providers that are defined in the project. | -------- | ------- | | `object` | `{}` | -### `${providers..config.*}` - -The resolved configuration for the provider. - -| Type | -| -------- | -| `object` | - -### `${providers..config.}` - -The provider config key value. Refer to individual [provider references](https://docs.garden.io/providers) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - -### `${providers..outputs.*}` - -The outputs defined by the provider (see individual plugin docs for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${providers..outputs.}` - -The provider output value. Refer to individual [provider references](https://docs.garden.io/providers) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${variables.*}` A map of all variables defined in the project configuration. @@ -753,14 +503,6 @@ A map of all variables defined in the project configuration. | -------- | ------- | | `object` | `{}` | -### `${variables.}` - -The value of the variable. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${var.*}` Alias for the variables field. @@ -769,14 +511,6 @@ Alias for the variables field. | -------- | ------- | | `object` | `{}` | -### `${var.}` - -Number, string or boolean - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${modules.*}` Retrieve information about modules that are defined in the project. @@ -785,64 +519,6 @@ Retrieve information about modules that are defined in the project. | -------- | ------- | | `object` | `{}` | -### `${modules..buildPath}` - -The build path of the module. - -| Type | -| -------- | -| `string` | - -Example: - -```yaml -my-variable: ${modules..buildPath} -``` - -### `${modules..outputs.*}` - -The outputs defined by the module (see individual module type [references](https://docs.garden.io/module-types) for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${modules..outputs.}` - -The module output value. Refer to individual [module type references](https://docs.garden.io/module-types) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - -### `${modules..path}` - -The local path of the module. - -| Type | -| -------- | -| `string` | - -Example: - -```yaml -my-variable: ${modules..path} -``` - -### `${modules..version}` - -The current version of the module. - -| Type | -| -------- | -| `string` | - -Example: - -```yaml -my-variable: ${modules..version} -``` - ### `${runtime.*}` Runtime outputs and information from services and tasks (only resolved at runtime when deploying services and running tasks). @@ -859,22 +535,6 @@ Runtime information from the services that the service/task being run depends on | -------- | ------- | | `object` | `{}` | -### `${runtime.services..outputs.*}` - -The runtime outputs defined by the service (see individual module type [references](https://docs.garden.io/module-types) for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${runtime.services..outputs.}` - -The service output value. Refer to individual [module type references](https://docs.garden.io/module-types) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - ### `${runtime.tasks.*}` Runtime information from the tasks that the service/task being run depends on. @@ -883,19 +543,3 @@ Runtime information from the tasks that the service/task being run depends on. | -------- | ------- | | `object` | `{}` | -### `${runtime.tasks..outputs.*}` - -The runtime outputs defined by the task (see individual module type [references](https://docs.garden.io/module-types) for details). - -| Type | Default | -| -------- | ------- | -| `object` | `{}` | - -### `${runtime.tasks..outputs.}` - -The task output value. Refer to individual [module type references](https://docs.garden.io/module-types) for details. - -| Type | -| --------------------------- | -| `number | string | boolean` | - diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json index 0124e7eb3e..ddc144d4b1 100644 --- a/garden-service/package-lock.json +++ b/garden-service/package-lock.json @@ -1475,11 +1475,11 @@ } }, "ajv": { - "version": "6.10.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", - "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.11.0.tgz", + "integrity": "sha512-nCprB/0syFYy9fVYU1ox1l2KN8S9I+tziH8D4zdZuLT3N6RMlGSGt5FSTpAiHB/Whv8Qs1cWHma1aMKZyaHRKA==", "requires": { - "fast-deep-equal": "^2.0.1", + "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" @@ -4656,9 +4656,9 @@ } }, "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==" }, "fast-diff": { "version": "1.2.0", @@ -4701,9 +4701,9 @@ } }, "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "fast-levenshtein": { "version": "2.0.6", diff --git a/garden-service/package.json b/garden-service/package.json index 2c8b6d52b9..2497e42e2f 100644 --- a/garden-service/package.json +++ b/garden-service/package.json @@ -26,6 +26,7 @@ "@hapi/joi": "https://github.com/garden-io/joi#master", "@kubernetes/client-node": "^0.11.0", "JSONStream": "^1.3.5", + "ajv": "^6.11.0", "analytics-node": "3.3.0", "ansi-escapes": "^4.3.0", "archiver": "^3.1.1", diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index b7db48cc88..2c3c655e24 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -9,7 +9,7 @@ import Bluebird = require("bluebird") import chalk from "chalk" -import { fromPairs, mapValues, omit, pickBy, keyBy } from "lodash" +import { fromPairs, mapValues, omit, pickBy, keyBy, uniqBy } from "lodash" import tmp from "tmp-promise" import cpy from "cpy" import normalizePath = require("normalize-path") @@ -97,6 +97,7 @@ import { relative, join } from "path" import { getArtifactKey } from "./util/artifacts" import { AugmentGraphResult, AugmentGraphParams } from "./types/plugin/provider/augmentGraph" import { DeployTask } from "./tasks/deploy" +import { BuildDependencyConfig } from "./config/module" const maxArtifactLogLines = 5 // max number of artifacts to list in console after task+test runs @@ -276,7 +277,19 @@ export class ActionRouter implements TypeGuard { ...params, } - const result = handler(handlerParams) + const result = await handler(handlerParams) + + // Consolidate the configured build dependencies, in case there are duplicates + const buildDeps: { [key: string]: BuildDependencyConfig } = {} + + for (const dep of result.moduleConfig.build.dependencies) { + if (buildDeps[dep.name]) { + buildDeps[dep.name].copy = uniqBy([...buildDeps[dep.name].copy, ...dep.copy], (c) => `${c.source}:${c.target}`) + } else { + buildDeps[dep.name] = dep + } + } + result.moduleConfig.build.dependencies = Object.values(buildDeps) this.garden.log.silly(`Called 'configure' handler for '${moduleType}'`) diff --git a/garden-service/src/config-graph.ts b/garden-service/src/config-graph.ts index 958c22a2da..62f7e2a848 100644 --- a/garden-service/src/config-graph.ts +++ b/garden-service/src/config-graph.ts @@ -8,7 +8,7 @@ import Bluebird from "bluebird" import toposort from "toposort" -import { flatten, pick, uniq, find, sortBy, pickBy } from "lodash" +import { flatten, pick, uniq, sortBy, pickBy } from "lodash" import { Garden } from "./garden" import { BuildDependencyConfig, ModuleConfig } from "./config/module" import { Module, getModuleKey, moduleFromConfig, moduleNeedsBuild } from "./types/module" @@ -127,7 +127,7 @@ export class ConfigGraph { // Make sure service source modules are added as build dependencies for the module const { sourceModuleName } = serviceConfig - if (sourceModuleName && !find(moduleConfig.build.dependencies, ["name", sourceModuleName])) { + if (sourceModuleName) { moduleConfig.build.dependencies.push({ name: sourceModuleName, copy: [], diff --git a/garden-service/src/config/base.ts b/garden-service/src/config/base.ts index d5766b6625..784b1b53a9 100644 --- a/garden-service/src/config/base.ts +++ b/garden-service/src/config/base.ts @@ -9,8 +9,8 @@ import { sep, resolve, relative, basename, dirname } from "path" import yaml from "js-yaml" import { readFile } from "fs-extra" -import { omit, isPlainObject, find } from "lodash" -import { ModuleResource, coreModuleSpecSchema, baseModuleSchemaKeys } from "./module" +import { omit, isPlainObject, find, isArray } from "lodash" +import { ModuleResource, coreModuleSpecSchema, baseModuleSchemaKeys, BuildDependencyConfig } from "./module" import { ConfigurationError } from "../exceptions" import { DEFAULT_API_VERSION } from "../constants" import { ProjectResource } from "../config/project" @@ -125,10 +125,11 @@ export function prepareModuleResource( * - foo-module * - name: foo-module // same as the above */ - const dependencies = - spec.build && spec.build.dependencies - ? spec.build.dependencies.map((dep) => (typeof dep === "string" ? { name: dep, copy: [] } : dep)) - : [] + let dependencies: BuildDependencyConfig[] = spec.build?.dependencies || [] + + if (spec.build && spec.build.dependencies && isArray(spec.build.dependencies)) { + dependencies = spec.build.dependencies.map((dep: any) => (typeof dep === "string" ? { name: dep, copy: [] } : dep)) + } // Built-in keys are validated here and the rest are put into the `spec` field const config: ModuleResource = { diff --git a/garden-service/src/config/common.ts b/garden-service/src/config/common.ts index 6ebfa9c015..4da435e48e 100644 --- a/garden-service/src/config/common.ts +++ b/garden-service/src/config/common.ts @@ -7,8 +7,13 @@ */ import Joi from "@hapi/joi" +import Ajv from "ajv" import { splitLast } from "../util/util" import { deline, dedent } from "../util/string" +import { cloneDeep } from "lodash" +import { joiPathPlaceholder } from "./validation" + +const ajv = new Ajv({ allErrors: true, useDefaults: true }) export type Primitive = string | number | boolean | null @@ -87,6 +92,10 @@ declare module "@hapi/joi" { } } +export interface CustomObjectSchema extends Joi.ObjectSchema { + jsonSchema(schema: object): this +} + export interface GitUrlSchema extends Joi.StringSchema { requireHash(): this } @@ -100,6 +109,7 @@ export interface PosixPathSchema extends Joi.StringSchema { } interface CustomJoi extends Joi.Root { + customObject: () => CustomObjectSchema gitUrl: () => GitUrlSchema posixPath: () => PosixPathSchema } @@ -200,7 +210,7 @@ export let joi: CustomJoi = Joi.extend({ }, }) -// We're supposed to be able to chain extend calls +// We're supposed to be able to chain extend calls, but the TS definitions are off joi = joi.extend({ base: Joi.string(), type: "gitUrl", @@ -236,6 +246,72 @@ joi = joi.extend({ }, }) +/** + * Add a joi.customObject() type, which includes additional methods, including one for validating with a + * JSON Schema. + * + * Note that the jsonSchema() option should generally not be used in conjunction with other options (like keys() + * and unknown()) since the behavior can be confusing. It is meant to facilitate a gradual transition away from Joi. + */ +joi = joi.extend({ + base: Joi.object(), + type: "customObject", + messages: { + validation: "", + }, + // TODO: check if jsonSchema() is being used in conjunction with other methods that may be incompatible. + // validate(value: string, { error }) { + // return { value } + // }, + rules: { + jsonSchema: { + method(jsonSchema: object) { + // tslint:disable-next-line: no-invalid-this + this.$_setFlag("jsonSchema", jsonSchema) + // tslint:disable-next-line: no-invalid-this + return this.$_addRule({ name: "jsonSchema", args: { jsonSchema } }) + }, + args: [ + { + name: "jsonSchema", + assert: (value) => { + return !!value + }, + message: "must be a valid JSON Schema with type=object", + normalize: (value) => { + if (value.type !== "object") { + return false + } + + try { + return ajv.compile(value) + } catch (err) { + return false + } + }, + }, + ], + validate(originalValue, helpers, args) { + const validate = args.jsonSchema + + // Need to do this to be able to assign defaults without mutating original value + const value = cloneDeep(originalValue) + const valid = validate(value) + + if (valid) { + return value + } else { + // TODO: customize the rendering here to make it a bit nicer + const errors = [...validate.errors] + const error = helpers.error("validation") + error.message = ajv.errorsText(errors, { dataVar: `value at ${joiPathPlaceholder}` }) + return error + } + }, + }, + }, +}) + export const joiPrimitive = () => joi .alternatives() diff --git a/garden-service/src/config/module.ts b/garden-service/src/config/module.ts index f4e72f2931..e6ab713cc6 100644 --- a/garden-service/src/config/module.ts +++ b/garden-service/src/config/module.ts @@ -232,7 +232,17 @@ export const moduleConfigSchema = baseModuleSpecSchema .description("The configuration for a module.") .unknown(false) -export const baseModuleSchemaKeys = Object.keys(moduleConfigSchema.describe().keys).concat(["kind"]) +export const baseModuleSchemaKeys = Object.keys(baseModuleSpecSchema.describe().keys).concat([ + "kind", + "name", + "type", + "path", + "configPath", + "serviceConfigs", + "taskConfigs", + "testConfigs", + "_ConfigType", +]) export function serializeConfig(moduleConfig: Partial) { return stableStringify(moduleConfig) diff --git a/garden-service/src/config/validation.ts b/garden-service/src/config/validation.ts index 9fd24c1ab3..bea391655d 100644 --- a/garden-service/src/config/validation.ts +++ b/garden-service/src/config/validation.ts @@ -12,7 +12,7 @@ import chalk from "chalk" import { relative } from "path" import uuid from "uuid" -const joiPathPlaceholder = uuid.v4() +export const joiPathPlaceholder = uuid.v4() const joiPathPlaceholderRegex = new RegExp(joiPathPlaceholder, "g") const errorPrefs: any = { wrap: { @@ -116,6 +116,8 @@ export function validateSchema( // a little hack to always use full key paths instead of just the label e.message = e.message.replace(joiLabelPlaceholderRegex, "key " + chalk.underline(renderedPath || ".")) e.message = e.message.replace(joiPathPlaceholderRegex, chalk.underline(renderedPath || ".")) + // FIXME: remove once we've customized the error output from AJV in customObject.jsonSchema() + e.message = e.message.replace(/should NOT have/g, "should not have") return e }) diff --git a/garden-service/src/docs/util.ts b/garden-service/src/docs/common.ts similarity index 68% rename from garden-service/src/docs/util.ts rename to garden-service/src/docs/common.ts index 890a17ce53..dfe6459a34 100644 --- a/garden-service/src/docs/util.ts +++ b/garden-service/src/docs/common.ts @@ -8,6 +8,33 @@ import { padEnd, max } from "lodash" +export interface NormalizedSchemaDescription { + type: string + name: string + allowedValuesOnly: boolean + allowedValues?: string + defaultValue?: string + deprecated: boolean + description?: string + experimental: boolean + formattedExample?: string + formattedName: string + formattedType: string + fullKey: string + hasChildren: boolean + internal: boolean + level: number + parent?: NormalizedSchemaDescription + required: boolean +} + +export interface NormalizeOptions { + level?: number + name?: string + parent?: NormalizedSchemaDescription + renderPatternKeys?: boolean +} + export function indent(lines: string[], level: number) { const prefix = padEnd("", level * 2, " ") return lines.map((line) => prefix + line) diff --git a/garden-service/src/docs/config.ts b/garden-service/src/docs/config.ts index f6704c75fc..4235edd818 100644 --- a/garden-service/src/docs/config.ts +++ b/garden-service/src/docs/config.ts @@ -12,172 +12,19 @@ import { safeDump } from "js-yaml" import linewrap from "linewrap" import { resolve } from "path" import { projectDocsSchema } from "../config/project" -import { get, flatten, uniq, find, isFunction, extend } from "lodash" +import { get, isFunction } from "lodash" import { baseModuleSpecSchema } from "../config/module" import handlebars = require("handlebars") import { joi } from "../config/common" import { GARDEN_SERVICE_ROOT } from "../constants" -import { indent, renderMarkdownTable, convertMarkdownLinks } from "./util" +import { indent, renderMarkdownTable, convertMarkdownLinks, NormalizedSchemaDescription } from "./common" +import { normalizeJoiSchemaDescription, JoiDescription } from "./joi-schema" export const TEMPLATES_DIR = resolve(GARDEN_SERVICE_ROOT, "src", "docs", "templates") const partialTemplatePath = resolve(TEMPLATES_DIR, "config-partial.hbs") const maxWidth = 120 -// Need this to fix the Joi typing -export interface Description extends Joi.Description { - name: string - level: number - parent?: NormalizedSchemaDescription - flags?: { - default?: any - description?: string - presence?: string - only?: boolean - } -} - -export interface NormalizedSchemaDescription extends Description { - required: boolean - defaultValue?: string - deprecated: boolean - fullKey: string - hasChildren: boolean - allowedValues?: string - description?: string - formattedExample?: string - formattedName: string - formattedType: string -} - -// Maps a Joi schema description into an array of descriptions and normalizes each entry. -// Filters out internal descriptions. -export function normalizeSchemaDescriptions( - joiDescription: Description, - { renderPatternKeys = false } = {} -): NormalizedSchemaDescription[] { - const normalize = ( - joiDesc: Description, - { level = 0, name, parent }: { level?: number; name?: string; parent?: NormalizedSchemaDescription } = {} - ): NormalizedSchemaDescription[] => { - let schemaDescription: NormalizedSchemaDescription | undefined - let childDescriptions: NormalizedSchemaDescription[] = [] - - // Skip descriptions without names since they merely point to the keys we're interested in. - // This means that we implicitly skip the first key of the schema. - if (name) { - schemaDescription = normalizeKeyDescription({ ...joiDesc, name, level, parent }) - } - - if (joiDesc.type === "object") { - const children = Object.entries(joiDesc.keys || {}) || [] - const nextLevel = name ? level + 1 : level - const nextParent = name ? schemaDescription : parent - childDescriptions = flatten( - children.map(([childName, childDescription]) => - normalize(childDescription as Description, { level: nextLevel, parent: nextParent, name: childName }) - ) - ) - if (renderPatternKeys && joiDesc.patterns && joiDesc.patterns.length > 0) { - const metas: any = extend({}, ...(joiDesc.metas || [])) - childDescriptions.push( - ...normalize(joiDesc.patterns[0].rule as Description, { - level: nextLevel, - parent: nextParent, - name: metas.keyPlaceholder || "", - }) - ) - } - } else if (joiDesc.type === "array") { - // We only use the first array item - const item = joiDesc.items[0] - childDescriptions = item ? normalize(item, { level: level + 2, parent: schemaDescription }) : [] - } - - if (!schemaDescription) { - return childDescriptions - } - return [schemaDescription, ...childDescriptions] - } - - return normalize(joiDescription).filter((key) => !get(key, "metas[0].internal")) -} - -// Normalizes the key description -function normalizeKeyDescription(schemaDescription: Description): NormalizedSchemaDescription { - const defaultValue = getDefaultValue(schemaDescription) - - let allowedValues: string | undefined = undefined - const allowOnly = schemaDescription.flags?.only === true - if (allowOnly) { - allowedValues = schemaDescription.allow!.map((v: any) => JSON.stringify(v)).join(", ") - } - - const presenceRequired = schemaDescription.flags?.presence === "required" - const required = presenceRequired || allowOnly - - let hasChildren: boolean = false - let arrayType: string | undefined - const { type } = schemaDescription - const formattedType = formatType(schemaDescription) - - const children = type === "object" && Object.entries(schemaDescription.keys || {}) - const items = type === "array" && schemaDescription.items - - if (children && children.length > 0) { - hasChildren = true - } else if (items && items.length > 0) { - // We don't consider an array of primitives as children - arrayType = items[0].type - hasChildren = arrayType === "array" || arrayType === "object" - } - - let formattedExample: string | undefined - if (schemaDescription.examples && schemaDescription.examples.length) { - const example = schemaDescription.examples[0] - if (schemaDescription.type === "object" || schemaDescription.type === "array") { - formattedExample = safeDump(example).trim() - } else { - formattedExample = JSON.stringify(example) - } - } - - const metas: any = extend({}, ...(schemaDescription.metas || [])) - const formattedName = type === "array" ? `${schemaDescription.name}[]` : schemaDescription.name - - const fullKey = schemaDescription.parent ? `${schemaDescription.parent.fullKey}.${formattedName}` : formattedName - - return { - ...schemaDescription, - deprecated: schemaDescription.parent?.deprecated || !!metas.deprecated, - description: schemaDescription.flags?.description, - fullKey, - formattedName, - formattedType, - defaultValue, - required, - allowedValues, - formattedExample, - hasChildren, - } -} - -function formatType(description: Description) { - const { type } = description - const items = type === "array" && description.items - - if (items && items.length > 0) { - // We don't consider an array of primitives as children - const arrayType = items[0].type - return `array[${arrayType}]` - } else if (type === "alternatives") { - // returns e.g. "string|number" - return uniq(description.matches.map(({ schema }) => formatType(schema))).join(" | ") - } else { - return type || "" - } -} - /** * Removes line starting with: # ``` */ @@ -185,12 +32,6 @@ export function sanitizeYamlStringForGitBook(yamlStr: string) { return yamlStr.replace(/.*# \`\`\`.*$\n/gm, "") } -export function getDefaultValue(schemaDescription: Description) { - const flags: any = schemaDescription.flags - const defaultSpec = flags?.default - return isFunction(defaultSpec) ? defaultSpec(schemaDescription.parent) : defaultSpec -} - function getParentDescriptions( schemaDescription: NormalizedSchemaDescription, schemaDescriptions: NormalizedSchemaDescription[] = [] @@ -211,10 +52,6 @@ export function renderMarkdownLink(description: NormalizedSchemaDescription) { function makeMarkdownDescription(description: NormalizedSchemaDescription, { showRequiredColumn = true } = {}) { const { formattedType, required, allowedValues, defaultValue, fullKey } = description - let experimentalFeature = false - if (description.meta) { - experimentalFeature = find(description.meta, (attr) => attr.experimental) || false - } const parentDescriptions = getParentDescriptions(description) const breadCrumbs = @@ -249,7 +86,7 @@ function makeMarkdownDescription(description: NormalizedSchemaDescription, { sho return { ...description, breadCrumbs, - experimentalFeature, + experimentalFeature: description.experimental, formattedExample, title: fullKey, table, @@ -493,7 +330,7 @@ interface RenderConfigOpts { * and a YAML schema. */ export function renderConfigReference(configSchema: Joi.ObjectSchema, { yamlOpts = {} }: RenderConfigOpts = {}) { - const normalizedDescriptions = normalizeSchemaDescriptions(configSchema.describe() as Description) + const normalizedDescriptions = normalizeJoiSchemaDescription(configSchema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(normalizedDescriptions, { renderBasicDescription: true, ...yamlOpts }) const keys = normalizedDescriptions.map((d) => makeMarkdownDescription(d)) @@ -516,14 +353,14 @@ export function renderTemplateStringReference({ placeholder?: string exampleName?: string }): string { - const normalizedSchemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description, { + const normalizedSchemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription, { renderPatternKeys: true, }) const keys = normalizedSchemaDescriptions .map((d) => makeMarkdownDescription(d, { showRequiredColumn: false })) // Omit objects without descriptions - .filter((d) => !(d.type === "object" && !d.flags?.description)) + .filter((d) => !(d.type === "object" && !d.description)) .map((d) => { let orgTitle = d.title diff --git a/garden-service/src/docs/joi-schema.ts b/garden-service/src/docs/joi-schema.ts new file mode 100644 index 0000000000..f37f441794 --- /dev/null +++ b/garden-service/src/docs/joi-schema.ts @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import Joi from "@hapi/joi" +import { safeDump } from "js-yaml" +import { flatten, uniq, isFunction, extend } from "lodash" +import { NormalizedSchemaDescription, NormalizeOptions } from "./common" +import { findByName } from "../util/util" +import { normalizeJsonSchema } from "./json-schema" + +// Need this to fix the Joi typing +export interface JoiDescription extends Joi.Description { + name: string + level: number + parent?: NormalizedSchemaDescription + flags?: { + default?: any + description?: string + presence?: string + only?: boolean + } +} + +// Maps a Joi schema description into an array of descriptions and normalizes each entry. +// Filters out internal descriptions. +export function normalizeJoiSchemaDescription( + joiDesc: JoiDescription, + { level = 0, name, parent, renderPatternKeys = false }: NormalizeOptions = {} +): NormalizedSchemaDescription[] { + let schemaDescription: NormalizedSchemaDescription | undefined + let childDescriptions: NormalizedSchemaDescription[] = [] + + // Skip descriptions without names since they merely point to the keys we're interested in. + // This means that we implicitly skip the first key of the schema. + if (name) { + schemaDescription = normalizeJoiKeyDescription({ ...joiDesc, name, level, parent }) + } + + if (joiDesc.type === "object" || joiDesc.type === "customObject") { + const children = Object.entries(joiDesc.keys || {}) || [] + const nextLevel = name ? level + 1 : level + const nextParent = name ? schemaDescription : parent + + childDescriptions = flatten( + children.map(([childName, childDescription]) => + normalizeJoiSchemaDescription(childDescription as JoiDescription, { + level: nextLevel, + parent: nextParent, + name: childName, + }) + ) + ) + + if (renderPatternKeys && joiDesc.patterns && joiDesc.patterns.length > 0) { + const metas: any = extend({}, ...(joiDesc.metas || [])) + childDescriptions.push( + ...normalizeJoiSchemaDescription(joiDesc.patterns[0].rule as JoiDescription, { + level: nextLevel, + parent: nextParent, + name: metas.keyPlaceholder || "", + }) + ) + } + + const jsonSchemaRule = findByName(joiDesc.rules || [], "jsonSchema") + + if (jsonSchemaRule) { + const jsonSchema = jsonSchemaRule.args.jsonSchema.schema + + childDescriptions.push( + ...flatten( + Object.entries(jsonSchema.properties).map(([childName, childDescription]) => + normalizeJsonSchema(childDescription as JoiDescription, { + level: nextLevel, + parent: nextParent, + name: childName, + }) + ) + ) + ) + } + } else if (joiDesc.type === "array") { + // We only use the first array item + const item = joiDesc.items[0] + childDescriptions = item ? normalizeJoiSchemaDescription(item, { level: level + 2, parent: schemaDescription }) : [] + } + + if (!schemaDescription) { + return childDescriptions + } + return [schemaDescription, ...childDescriptions].filter((key) => !key.internal) +} + +// Normalizes the key description +function normalizeJoiKeyDescription(schemaDescription: JoiDescription): NormalizedSchemaDescription { + const defaultValue = getJoiDefaultValue(schemaDescription) + + let allowedValues: string | undefined = undefined + const allowOnly = schemaDescription.flags?.only === true + if (allowOnly) { + allowedValues = schemaDescription.allow!.map((v: any) => JSON.stringify(v)).join(", ") + } + + const presenceRequired = schemaDescription.flags?.presence === "required" + const required = presenceRequired || allowOnly + + let hasChildren: boolean = false + let arrayType: string | undefined + const { type } = schemaDescription + const formattedType = formatType(schemaDescription) + + const children = type === "object" && Object.entries(schemaDescription.keys || {}) + const items = type === "array" && schemaDescription.items + + if (children && children.length > 0) { + hasChildren = true + } else if (items && items.length > 0) { + // We don't consider an array of primitives as children + arrayType = items[0].type + hasChildren = arrayType === "array" || arrayType === "object" + } + + let formattedExample: string | undefined + if (schemaDescription.examples && schemaDescription.examples.length) { + const example = schemaDescription.examples[0] + if (schemaDescription.type === "object" || schemaDescription.type === "array") { + formattedExample = safeDump(example).trim() + } else { + formattedExample = JSON.stringify(example) + } + } + + const metas: any = extend({}, ...(schemaDescription.metas || [])) + const formattedName = type === "array" ? `${schemaDescription.name}[]` : schemaDescription.name + + const fullKey = schemaDescription.parent ? `${schemaDescription.parent.fullKey}.${formattedName}` : formattedName + + return { + type: type!, + name: schemaDescription.name, + allowedValues, + allowedValuesOnly: !!schemaDescription.flags?.only, + defaultValue, + deprecated: schemaDescription.parent?.deprecated || !!metas.deprecated, + description: schemaDescription.flags?.description, + experimental: schemaDescription.parent?.experimental || !!metas.experimental, + fullKey, + formattedExample, + formattedName, + formattedType, + hasChildren, + internal: schemaDescription.parent?.internal || !!metas.internal, + level: schemaDescription.level, + parent: schemaDescription.parent, + required, + } +} + +export function getJoiDefaultValue(schemaDescription: JoiDescription) { + const flags: any = schemaDescription.flags + const defaultSpec = flags?.default + return isFunction(defaultSpec) ? defaultSpec(schemaDescription.parent) : defaultSpec +} + +function formatType(description: JoiDescription) { + const { type } = description + const items = type === "array" && description.items + + if (items && items.length > 0) { + // We don't consider an array of primitives as children + const arrayType = items[0].type + return `array[${arrayType}]` + } else if (type === "alternatives") { + // returns e.g. "string|number" + return uniq(description.matches.map(({ schema }) => formatType(schema))).join(" | ") + } else { + return type || "" + } +} diff --git a/garden-service/src/docs/json-schema.ts b/garden-service/src/docs/json-schema.ts new file mode 100644 index 0000000000..9a4bdfd27d --- /dev/null +++ b/garden-service/src/docs/json-schema.ts @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { safeDump } from "js-yaml" +import { flatten, isArray } from "lodash" +import { NormalizedSchemaDescription, NormalizeOptions } from "./common" +import { ValidationError } from "../exceptions" + +/** + * Takes a JSON Schema and translates to a list of NormalizedKeyDescription objects. + * Analogous to normalizeJoiSchemaDescription(), and flows the same way. + */ +export function normalizeJsonSchema( + schema: any, + { level = 0, name, parent, renderPatternKeys = false }: NormalizeOptions = {} +): NormalizedSchemaDescription[] { + let schemaDescription: NormalizedSchemaDescription | undefined + let childDescriptions: NormalizedSchemaDescription[] = [] + + // Skip descriptions without names since they merely point to the keys we're interested in. + // This means that we implicitly skip the first key of the schema. + if (name) { + schemaDescription = normalizeJsonKeyDescription(schema, { name, level, parent }) + } + + const type = getType(schema) + + if (type === "object") { + const children = Object.entries(schema.properties || {}) || [] + const nextLevel = name ? level + 1 : level + const nextParent = name ? schemaDescription : parent + + childDescriptions = flatten( + children.map(([childName, childSchema]) => + normalizeJsonSchema(childSchema, { level: nextLevel, parent: nextParent, name: childName }) + ) + ) + + if (renderPatternKeys && schema.patterns && schema.patterns.length > 0) { + // TODO: implement pattern schemas + } + } else if (type === "array") { + // We only use the first array item + const item = schema.items[0] + childDescriptions = item ? normalizeJsonSchema(item, { level: level + 2, parent: schemaDescription }) : [] + } + + if (!schemaDescription) { + return childDescriptions + } + return [schemaDescription, ...childDescriptions].filter((key) => !key.internal) +} + +// Normalizes the key description. +// TODO: This no doubt requires more work. Just implementing the bare necessities for our currently configured schemas. +function normalizeJsonKeyDescription( + schema: any, + { + level, + name, + parent, + parentSchema, + }: { level: number; name: string; parent?: NormalizedSchemaDescription; parentSchema?: any } +): NormalizedSchemaDescription { + let allowedValues: string[] | undefined + + if (isArray(schema.type) && schema.type.includes(null)) { + allowedValues = ["null"] + } + + const type = getType(schema) + + if (!type) { + throw new ValidationError(`Missing type property on JSON Schema`, { schema }) + } + + const formattedName = type === "array" ? `${name}[]` : name + + let formattedExample: string | undefined + if (schema.examples && schema.examples.length > 0) { + const example = schema.examples[0] + if (type === "object" || type === "array") { + formattedExample = safeDump(example).trim() + } else { + formattedExample = JSON.stringify(example) + } + } + + const output: NormalizedSchemaDescription = { + type, + name, + allowedValuesOnly: false, + defaultValue: schema.default, + deprecated: !!schema.deprecated, + description: schema.description, + experimental: !!schema["x-garden-experimental"], + fullKey: parent ? `${parent.fullKey}.${formattedName}` : formattedName, + formattedExample, + formattedName, + formattedType: formatType(schema), + hasChildren: false, + internal: !!schema["x-garden-internal"], + level, + parent, + required: false, + } + + if (schema.enum) { + output.allowedValuesOnly = true + allowedValues = [...(allowedValues || []), ...schema.enum.map((v: any) => JSON.stringify(v))] + } + + if (allowedValues) { + output.allowedValues = allowedValues?.join(", ") + } + + if (parent?.type === "object" && parentSchema?.required.includes(name)) { + output.required = true + } + + let arrayType: string | undefined + + const children = type === "object" && Object.entries(schema.properties || {}) + const items = type === "array" && schema.items + + if (children && children.length > 0) { + output.hasChildren = true + } else if (items && items.length > 0) { + // We don't consider an array of primitives as children + arrayType = items[0].type + output.hasChildren = arrayType === "array" || arrayType === "object" + } + + return output +} + +function getType(schema: any): string { + const { type } = schema + + if (isArray(type)) { + // TODO: handle multiple type options + return type.filter((t) => t !== null)[0] + } else { + return type + } +} + +function formatType(schema: any) { + const type = getType(schema) + const items = type === "array" && schema.items + + if (items && items.length > 0) { + const arrayType = items[0].type + return `array[${arrayType}]` + } else { + return type || "" + } +} diff --git a/garden-service/src/docs/module-type.ts b/garden-service/src/docs/module-type.ts index b09832022e..5166f8bfb0 100644 --- a/garden-service/src/docs/module-type.ts +++ b/garden-service/src/docs/module-type.ts @@ -26,7 +26,8 @@ export const moduleTypes = [ { name: "helm", pluginName: "local-kubernetes" }, { name: "kubernetes", pluginName: "local-kubernetes" }, { name: "maven-container" }, - { name: "openfaas", pluginName: "local-kubernetes" }, + { name: "openfaas" }, + { name: "persistentvolumeclaim", pluginName: "local-kubernetes" }, { name: "terraform" }, ] diff --git a/garden-service/src/plugins/base-volume.ts b/garden-service/src/plugins/base-volume.ts new file mode 100644 index 0000000000..58d1369860 --- /dev/null +++ b/garden-service/src/plugins/base-volume.ts @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { baseModuleSpecSchema, ModuleSpec } from "../config/module" +import { joi } from "../config/common" +import { dedent } from "../util/string" +import { createGardenPlugin } from "../types/plugin/plugin" + +type VolumeAccessMode = "ReadOnlyMany" | "ReadWriteOnce" | "ReadWriteMany" + +export interface BaseVolumeSpec extends ModuleSpec { + accessModes: VolumeAccessMode[] +} + +export const baseVolumeSpecSchema = baseModuleSpecSchema.keys({ + accessModes: joi + .array() + .items(joi.string().allow("ReadOnlyMany", "ReadWriteOnce", "ReadWriteMany")) + .required() + .unique() + .min(1).description(dedent` + A list of access modes supported by the volume when mounting. At least one must be specified. The available modes are as follows: + + ReadOnlyMany - May be mounted as a read-only volume, concurrently by multiple targets. + ReadWriteOnce - May be mounted as a read-write volume by a single target at a time. + ReadWriteMany - May be mounted as a read-write volume, concurrently by multiple targets. + + At least one mode must be specified. + `), +}) + +export const gardenPlugin = createGardenPlugin({ + name: "base-volume", + createModuleTypes: [ + { + name: "base-volume", + docs: dedent` + Internal abstraction used for specifying and referencing (usually persistent) volumes by other module types. + `, + schema: baseVolumeSpecSchema, + handlers: {}, + }, + ], +}) diff --git a/garden-service/src/plugins/container/config.ts b/garden-service/src/plugins/container/config.ts index 04b6f46da8..ff551d934a 100644 --- a/garden-service/src/plugins/container/config.ts +++ b/garden-service/src/plugins/container/config.ts @@ -18,10 +18,11 @@ import { envVarRegex, Primitive, joiModuleIncludeDirective, + joiIdentifier, } from "../../config/common" import { ArtifactSpec } from "./../../config/validation" import { Service, ingressHostnameSchema, linkUrlSchema } from "../../types/service" -import { DEFAULT_PORT_PROTOCOL } from "../../constants" +import { DEFAULT_PORT_PROTOCOL, DOCS_BASE_URL } from "../../constants" import { ModuleSpec, ModuleConfig, baseBuildSpecSchema, BaseBuildSpec } from "../../config/module" import { CommonServiceSpec, ServiceConfig, baseServiceSpecSchema } from "../../config/service" import { baseTaskSpecSchema, BaseTaskSpec, cacheResultSchema } from "../../config/task" @@ -54,10 +55,11 @@ export interface ServicePortSpec { nodePort?: number | true } -export interface ServiceVolumeSpec { +export interface ContainerVolumeSpec { name: string containerPath: string hostPath?: string + module?: string } export interface ServiceHealthCheckSpec { @@ -92,7 +94,7 @@ export interface ContainerServiceSpec extends CommonServiceSpec { limits: ServiceLimitSpec ports: ServicePortSpec[] replicas?: number - volumes: ServiceVolumeSpec[] + volumes: ContainerVolumeSpec[] } export const commandExample = ["/bin/sh", "-c"] @@ -307,29 +309,49 @@ export const portSchema = joi.object().keys({ `), }) -const volumeSchema = joi.object().keys({ - name: joiUserIdentifier() - .required() - .description("The name of the allocated volume."), - containerPath: joi - .posixPath() - .required() - .description("The path where the volume should be mounted in the container."), - hostPath: joi - .posixPath() - .description( - dedent` +const volumeSchema = joi + .object() + .keys({ + name: joiUserIdentifier() + .required() + .description("The name of the allocated volume."), + containerPath: joi + .posixPath() + .required() + .description("The path where the volume should be mounted in the container."), + hostPath: joi + .posixPath() + .description( + dedent` _NOTE: Usage of hostPath is generally discouraged, since it doesn't work reliably across different platforms and providers. Some providers may not support it at all._ A local path or path on the node that's running the container, to mount in the container, relative to the module source path (or absolute). ` - ) - .example("/some/dir"), -}) + ) + .example("/some/dir"), + module: joiIdentifier().description( + dedent` + The name of a _volume module_ that should be mounted at \`containerPath\`. The supported module types will depend on which provider you are using. The \`kubernetes\` provider supports the [persistentvolumeclaim module](${DOCS_BASE_URL}/module-types/persistentvolumeclaim), for example. + + When a \`module\` is specified, the referenced module/volume will be automatically configured as a runtime dependency of this service, as well as a build dependency of this module. + + Note: Make sure to pay attention to the supported \`accessModes\` of the referenced volume. Unless it supports the ReadWriteMany access mode, you'll need to make sure it is not configured to be mounted by multiple services at the same time. Refer to the documentation of the module type in question to learn more. + ` + ), + }) + .oxor("hostPath", "module") -const serviceSchema = baseServiceSpecSchema.keys({ +export function getContainerVolumesSchema(targetType: string) { + return joiArray(volumeSchema).unique("name").description(dedent` + List of volumes that should be mounted when deploying the ${targetType}. + + Note: If neither \`hostPath\` nor \`module\` is specified, an empty ephemeral volume is created and mounted when deploying the container. + `) +} + +const containerServiceSchema = baseServiceSpecSchema.keys({ annotations: annotationsSchema.description( "Annotations to attach to the service (Note: May not be applicable to all providers)." ), @@ -381,9 +403,7 @@ const serviceSchema = baseServiceSpecSchema.keys({ Note: This setting may be overridden or ignored in some cases. For example, when running with \`daemon: true\`, with hot-reloading enabled, or if the provider doesn't support multiple replicas. `), - volumes: joiArray(volumeSchema) - .unique("name") - .description("List of volumes that should be mounted when deploying the container."), + volumes: getContainerVolumesSchema("service"), }) export interface ContainerRegistryConfig { @@ -450,6 +470,7 @@ export interface ContainerTestSpec extends BaseTestSpec { artifacts: ArtifactSpec[] command?: string[] env: ContainerEnvVars + volumes: ContainerVolumeSpec[] } export const containerTestSchema = baseTestSpecSchema.keys({ @@ -465,6 +486,7 @@ export const containerTestSchema = baseTestSpecSchema.keys({ .description("The command/entrypoint used to run the test inside the container.") .example(commandExample), env: containerEnvVarsSchema, + volumes: getContainerVolumesSchema("test"), }) export interface ContainerTaskSpec extends BaseTaskSpec { @@ -473,6 +495,7 @@ export interface ContainerTaskSpec extends BaseTaskSpec { cacheResult: boolean command?: string[] env: ContainerEnvVars + volumes: ContainerVolumeSpec[] } export const containerTaskSchema = baseTaskSpecSchema @@ -490,6 +513,7 @@ export const containerTaskSchema = baseTaskSpecSchema .description("The command/entrypoint used to run the task inside the container.") .example(commandExample), env: containerEnvVarsSchema, + volumes: getContainerVolumesSchema("task"), }) .description("A task that can be run in the container.") @@ -556,7 +580,7 @@ export const containerModuleSpecSchema = joi .posixPath() .subPathOnly() .description("POSIX-style name of Dockerfile, relative to module root."), - services: joiArray(serviceSchema) + services: joiArray(containerServiceSchema) .unique("name") .description("A list of services to deploy from this container module."), tests: joiArray(containerTestSchema).description("A list of tests to run in the module."), diff --git a/garden-service/src/plugins/container/container.ts b/garden-service/src/plugins/container/container.ts index 203722f3ae..5bfc683928 100644 --- a/garden-service/src/plugins/container/container.ts +++ b/garden-service/src/plugins/container/container.ts @@ -6,7 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import dedent = require("dedent") import chalk from "chalk" import { keyBy } from "lodash" @@ -23,6 +22,7 @@ import { publishContainerModule } from "./publish" import { DOCS_BASE_URL } from "../../constants" import { SuggestModulesParams, SuggestModulesResult } from "../../types/plugin/module/suggestModules" import { listDirectory } from "../../util/fs" +import { dedent } from "../../util/string" export const containerModuleOutputsSchema = joi.object().keys({ "local-image-name": joi @@ -121,6 +121,13 @@ export async function configureContainerModule({ ctx, log, moduleConfig }: Confi } } + for (const volume of spec.volumes) { + if (volume.module) { + moduleConfig.build.dependencies.push({ name: volume.module, copy: [] }) + spec.dependencies.push(volume.module) + } + } + return { name, dependencies: spec.dependencies, @@ -130,22 +137,40 @@ export async function configureContainerModule({ ctx, log, moduleConfig }: Confi } }) - moduleConfig.testConfigs = moduleConfig.spec.tests.map((t) => ({ - name: t.name, - dependencies: t.dependencies, - disabled: t.disabled, - spec: t, - timeout: t.timeout, - })) - - moduleConfig.taskConfigs = moduleConfig.spec.tasks.map((t) => ({ - name: t.name, - cacheResult: t.cacheResult, - dependencies: t.dependencies, - disabled: t.disabled, - spec: t, - timeout: t.timeout, - })) + moduleConfig.testConfigs = moduleConfig.spec.tests.map((t) => { + for (const volume of t.volumes) { + if (volume.module) { + moduleConfig.build.dependencies.push({ name: volume.module, copy: [] }) + t.dependencies.push(volume.module) + } + } + + return { + name: t.name, + dependencies: t.dependencies, + disabled: t.disabled, + spec: t, + timeout: t.timeout, + } + }) + + moduleConfig.taskConfigs = moduleConfig.spec.tasks.map((t) => { + for (const volume of t.volumes) { + if (volume.module) { + moduleConfig.build.dependencies.push({ name: volume.module, copy: [] }) + t.dependencies.push(volume.module) + } + } + + return { + name: t.name, + cacheResult: t.cacheResult, + dependencies: t.dependencies, + disabled: t.disabled, + spec: t, + timeout: t.timeout, + } + }) const provider = ctx.provider const deploymentImageName = await containerHelpers.getDeploymentImageName( diff --git a/garden-service/src/plugins/kubernetes/container/deployment.ts b/garden-service/src/plugins/kubernetes/container/deployment.ts index da5cfd1cf1..266e393467 100644 --- a/garden-service/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/src/plugins/kubernetes/container/deployment.ts @@ -7,10 +7,10 @@ */ import chalk from "chalk" -import { V1Container, V1Affinity } from "@kubernetes/client-node" +import { V1Container, V1Affinity, V1VolumeMount, V1PodSpec } from "@kubernetes/client-node" import { Service } from "../../../types/service" import { extend, find, keyBy, merge, set } from "lodash" -import { ContainerModule, ContainerService } from "../../container/config" +import { ContainerModule, ContainerService, ContainerVolumeSpec } from "../../container/config" import { createIngressResources } from "./ingress" import { createServiceResources } from "./service" import { waitForResources, compareDeployedResources } from "../status/status" @@ -28,7 +28,7 @@ import { LogEntry } from "../../../logger/log-entry" import { DeployServiceParams } from "../../../types/plugin/service/deployService" import { DeleteServiceParams } from "../../../types/plugin/service/deleteService" import { millicpuToString, kilobytesToString, prepareEnvVars, workloadTypes } from "../util" -import { gardenAnnotationKey } from "../../../util/string" +import { gardenAnnotationKey, deline } from "../../../util/string" import { RuntimeContext } from "../../../runtime-context" import { resolve } from "path" import { killPortForwards } from "../port-forward" @@ -317,6 +317,8 @@ export async function createWorkloadManifest({ }, } + deployment.spec.template.spec.containers = [container] + if (service.spec.command && service.spec.command.length > 0) { container.command = service.spec.command } @@ -330,7 +332,7 @@ export async function createWorkloadManifest({ } if (spec.volumes && spec.volumes.length) { - configureVolumes(service.module, deployment, container, spec) + configureVolumes(service.module, deployment.spec.template.spec, spec.volumes) } const ports = spec.ports @@ -384,8 +386,6 @@ export async function createWorkloadManifest({ delete container.ports } - deployment.spec.template.spec.containers = [container] - if (production) { const affinity: V1Affinity = { podAntiAffinity: { @@ -544,45 +544,63 @@ function configureHealthCheck(container, spec): void { } } -function configureVolumes(module: ContainerModule, deployment, container, spec): void { +export function configureVolumes( + module: ContainerModule, + podSpec: V1PodSpec, + volumeSpecs: ContainerVolumeSpec[] +): void { const volumes: any[] = [] - const volumeMounts: any[] = [] + const volumeMounts: V1VolumeMount[] = [] - for (const volume of spec.volumes) { + for (const volume of volumeSpecs) { const volumeName = volume.name - const volumeType = !!volume.hostPath ? "hostPath" : "emptyDir" if (!volumeName) { throw new Error("Must specify volume name") } - if (volumeType === "emptyDir") { - volumes.push({ - name: volumeName, - emptyDir: {}, - }) - volumeMounts.push({ - name: volumeName, - mountPath: volume.containerPath, - }) - } else if (volumeType === "hostPath") { + volumeMounts.push({ + name: volumeName, + mountPath: volume.containerPath, + }) + + if (volume.hostPath) { volumes.push({ name: volumeName, hostPath: { path: resolve(module.path, volume.hostPath), }, }) - volumeMounts.push({ + } else if (volume.module) { + // Make sure the module is a supported type + const volumeModule = module.buildDependencies[volume.module] + + if (!volumeModule.compatibleTypes.includes("persistentvolumeclaim")) { + throw new ConfigurationError( + chalk.red(deline`Container module ${chalk.white(module.name)} specifies a unsupported module + ${chalk.white(volumeModule.name)} for volume mount ${chalk.white(volumeName)}. Only persistentvolumeclaim + modules are supported at this time. + `), + { volumeSpec: volume } + ) + } + + volumes.push({ name: volumeName, - mountPath: volume.containerPath, + persistentVolumeClaim: { + claimName: volume.module, + }, }) } else { - throw new Error("Unsupported volume type: " + volumeType) + volumes.push({ + name: volumeName, + emptyDir: {}, + }) } } - deployment.spec.template.spec.volumes = volumes - container.volumeMounts = volumeMounts + podSpec.volumes = volumes + podSpec.containers[0].volumeMounts = volumeMounts } /** diff --git a/garden-service/src/plugins/kubernetes/container/run.ts b/garden-service/src/plugins/kubernetes/container/run.ts index 97f20f4703..4fcf1d2662 100644 --- a/garden-service/src/plugins/kubernetes/container/run.ts +++ b/garden-service/src/plugins/kubernetes/container/run.ts @@ -75,6 +75,7 @@ export async function runContainerTask(params: RunTaskParams): podName: makePodName("task", module.name, task.name), description: `Task '${task.name}' in container module '${module.name}'`, timeout: task.spec.timeout || undefined, + volumes: task.spec.volumes, }) const result: RunTaskResult = { diff --git a/garden-service/src/plugins/kubernetes/container/test.ts b/garden-service/src/plugins/kubernetes/container/test.ts index c9f0d08b66..8a273395cd 100644 --- a/garden-service/src/plugins/kubernetes/container/test.ts +++ b/garden-service/src/plugins/kubernetes/container/test.ts @@ -38,6 +38,7 @@ export async function testContainerModule(params: TestModuleParams { if (spec.resource && spec.resource.containerModule) { - addBuildDependency(spec.resource.containerModule) + moduleConfig.build.dependencies.push({ name: spec.resource.containerModule, copy: [] }) } return { @@ -283,7 +267,7 @@ export async function configureHelmModule({ moduleConfig.testConfigs = tests.map((spec) => { if (spec.resource && spec.resource.containerModule) { - addBuildDependency(spec.resource.containerModule) + moduleConfig.build.dependencies.push({ name: spec.resource.containerModule, copy: [] }) } return { diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts index ea5d089559..6b1c46c575 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/config.ts @@ -11,7 +11,6 @@ import { joiArray, joi, joiModuleIncludeDirective } from "../../../config/common import { Module } from "../../../types/module" import { ConfigureModuleParams, ConfigureModuleResult } from "../../../types/plugin/module/configure" import { Service } from "../../../types/service" -import { ContainerModule } from "../../container/config" import { baseBuildSpecSchema } from "../../../config/module" import { KubernetesResource } from "../types" import { deline, dedent } from "../../../util/string" @@ -41,7 +40,7 @@ export interface KubernetesServiceSpec { tests: KubernetesTestSpec[] } -export type KubernetesService = Service +export type KubernetesService = Service const kubernetesResourceSchema = joi .object() diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts index 46dd214a17..913aa14cbf 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts @@ -35,10 +35,10 @@ export const kubernetesHandlers: Partial): Prom return { fresh: true } } -async function getServiceStatus({ +export async function getKubernetesServiceStatus({ ctx, module, log, @@ -88,7 +88,9 @@ async function getServiceStatus({ } } -async function deployService(params: DeployServiceParams): Promise { +export async function deployKubernetesService( + params: DeployServiceParams +): Promise { const { ctx, module, service, log } = params const k8sCtx = ctx @@ -114,7 +116,7 @@ async function deployService(params: DeployServiceParams): Pro log, }) - const status = await getServiceStatus(params) + const status = await getKubernetesServiceStatus(params) // Make sure port forwards work after redeployment killPortForwards(service, status.forwardablePorts || [], log) diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index f6e30b75b9..c05eb47e89 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -36,6 +36,7 @@ import { getSystemMetadataNamespaceName } from "./system" import { removeTillerCmd } from "./commands/remove-tiller" import { DOCS_BASE_URL } from "../../constants" import { inClusterRegistryHostname } from "./constants" +import { pvcModuleDefinition } from "./volumes/persistentvolumeclaim" export async function configureProvider({ projectName, @@ -225,6 +226,7 @@ export const gardenPlugin = createGardenPlugin({ schema: kubernetesModuleSpecSchema, handlers: kubernetesHandlers, }, + pvcModuleDefinition, ], extendModuleTypes: [ { diff --git a/garden-service/src/plugins/kubernetes/run.ts b/garden-service/src/plugins/kubernetes/run.ts index c41d9f2393..eb43b7df13 100644 --- a/garden-service/src/plugins/kubernetes/run.ts +++ b/garden-service/src/plugins/kubernetes/run.ts @@ -25,12 +25,13 @@ import { checkPodStatus, getPodLogs } from "./status/pod" import { KubernetesServerResource } from "./types" import { ServiceState } from "../../types/service" import { RunModuleParams } from "../../types/plugin/module/runModule" -import { ContainerEnvVars } from "../container/config" +import { ContainerEnvVars, ContainerVolumeSpec } from "../container/config" import { prepareEnvVars, makePodName } from "./util" import { deline } from "../../util/string" import { ArtifactSpec } from "../../config/validation" import cpy from "cpy" import { prepareImagePullSecrets } from "./secrets" +import { configureVolumes } from "./container/deployment" export async function runAndCopy({ ctx, @@ -51,6 +52,7 @@ export async function runAndCopy({ stdout, stderr, namespace, + volumes, }: RunModuleParams & { image: string container?: V1Container @@ -62,6 +64,7 @@ export async function runAndCopy({ stdout?: Writable stderr?: Writable namespace: string + volumes?: ContainerVolumeSpec[] }): Promise { const provider = ctx.provider const api = await KubeApi.factory(log, provider) @@ -92,6 +95,10 @@ export async function runAndCopy({ imagePullSecrets: await prepareImagePullSecrets({ api, provider, namespace, log }), } + if (volumes) { + configureVolumes(module, spec, volumes) + } + if (!description) { description = `Container module '${module.name}'` } diff --git a/garden-service/src/plugins/kubernetes/status/status.ts b/garden-service/src/plugins/kubernetes/status/status.ts index efe1ff4490..9b13970b34 100644 --- a/garden-service/src/plugins/kubernetes/status/status.ts +++ b/garden-service/src/plugins/kubernetes/status/status.ts @@ -54,6 +54,13 @@ interface ObjHandler { (params: StatusHandlerParams): Promise } +const pvcPhaseMap: { [key: string]: ServiceState } = { + Available: "deploying", + Bound: "ready", + Released: "stopped", + Failed: "unhealthy", +} + // Handlers to check the rollout status for K8s objects where that applies. // Using https://github.com/kubernetes/helm/blob/master/pkg/kube/wait.go as a reference here. const objHandlers: { [kind: string]: ObjHandler } = { @@ -63,7 +70,7 @@ const objHandlers: { [kind: string]: ObjHandler } = { PersistentVolumeClaim: async ({ resource }) => { const pvc = >resource - const state: ServiceState = pvc.status.phase === "Bound" ? "ready" : "deploying" + const state: ServiceState = pvcPhaseMap[pvc.status.phase!] || "unknown" return { state, resource } }, diff --git a/garden-service/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts b/garden-service/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts new file mode 100644 index 0000000000..3cd054cda3 --- /dev/null +++ b/garden-service/src/plugins/kubernetes/volumes/persistentvolumeclaim.ts @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { joiIdentifier, joi } from "../../../config/common" +import { dedent } from "../../../util/string" +import { BaseVolumeSpec } from "../../base-volume" +import { V1PersistentVolumeClaimSpec, V1PersistentVolumeClaim } from "@kubernetes/client-node" +import { readFileSync } from "fs-extra" +import { join } from "path" +import { ModuleTypeDefinition } from "../../../types/plugin/plugin" +import { DOCS_BASE_URL, STATIC_DIR } from "../../../constants" +import { baseBuildSpecSchema } from "../../../config/module" +import { ConfigureModuleParams } from "../../../types/plugin/module/configure" +import { GetServiceStatusParams } from "../../../types/plugin/service/getServiceStatus" +import { Module } from "../../../types/module" +import { KubernetesModule, KubernetesModuleConfig, KubernetesService } from "../kubernetes-module/config" +import { KubernetesResource } from "../types" +import { getKubernetesServiceStatus, deployKubernetesService } from "../kubernetes-module/handlers" +import { DeployServiceParams } from "../../../types/plugin/service/deployService" + +export interface PersistentVolumeClaimSpec extends BaseVolumeSpec { + dependencies: string[] + namespace: string + spec: V1PersistentVolumeClaimSpec +} + +type PersistentVolumeClaimModule = Module + +// Need to use a sync read to avoid having to refactor createGardenPlugin() +// The `persistentvolumeclaim.json` file is copied from the handy +// kubernetes-json-schema repo (https://github.com/instrumenta/kubernetes-json-schema/tree/master/v1.17.0-standalone). +const jsonSchema = JSON.parse(readFileSync(join(STATIC_DIR, "kubernetes", "persistentvolumeclaim.json")).toString()) + +export const pvcModuleDefinition: ModuleTypeDefinition = { + name: "persistentvolumeclaim", + docs: dedent` + Creates a [PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) in your namespace, that can be referenced and mounted by other resources and [container modules](${DOCS_BASE_URL}/module-types/container). + + See the [Mounting volumes](${DOCS_BASE_URL}/guides/container-modules#mounting-volumes) guide for more info and usage examples. + `, + schema: joi.object().keys({ + build: baseBuildSpecSchema, + dependencies: joiIdentifier().description("List of services and tasks to deploy/run before deploying this PVC."), + namespace: joiIdentifier().description( + "The namespace to deploy the PVC in. Note that any module referencing the PVC must be in the same namespace, so in most cases you should leave this unset." + ), + spec: joi + .customObject() + .jsonSchema({ ...jsonSchema.properties.spec, type: "object" }) + .required() + .description( + "The spec for the PVC. This is passed directly to the created PersistentVolumeClaim resource. Note that the spec schema may include (or even require) additional fields, depending on the used `storageClass`. See the [PersistentVolumeClaim docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) for details." + ), + }), + handlers: { + async configure({ moduleConfig }: ConfigureModuleParams) { + // No need to scan for files + moduleConfig.include = [] + + // Copy the access modes field to match the BaseVolumeSpec schema + moduleConfig.spec.accessModes = moduleConfig.spec.spec.accessModes + + moduleConfig.serviceConfigs = [ + { + dependencies: moduleConfig.spec.dependencies, + disabled: moduleConfig.spec.disabled, + hotReloadable: false, + name: moduleConfig.name, + spec: moduleConfig.spec, + }, + ] + + return { moduleConfig } + }, + + async getServiceStatus(params: GetServiceStatusParams) { + params.service = getKubernetesService(params.module) + params.module = params.service.module + + return getKubernetesServiceStatus(params) + }, + + async deployService(params: DeployServiceParams) { + params.service = getKubernetesService(params.module) + params.module = params.service.module + + return deployKubernetesService(params) + }, + }, +} + +/** + * Maps a `persistentvolumeclaim` module to a `kubernetes` module (so we can re-use those handlers). + */ +function getKubernetesService(pvcModule: PersistentVolumeClaimModule): KubernetesService { + const pvcManifest: KubernetesResource = { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: pvcModule.name, + }, + spec: pvcModule.spec.spec, + } + + const spec = { + dependencies: pvcModule.spec.dependencies, + files: [], + manifests: [pvcManifest], + tasks: [], + tests: [], + } + + const serviceConfig = { + ...pvcModule.serviceConfigs[0], + spec, + } + + const config: KubernetesModuleConfig = { + ...pvcModule, + serviceConfigs: [serviceConfig], + spec, + taskConfigs: [], + testConfigs: [], + } + + const module: KubernetesModule = { + ...pvcModule, + _ConfigType: config, + ...config, + spec: { + ...pvcModule.spec, + files: [], + manifests: [pvcManifest], + tasks: [], + tests: [], + }, + } + + return { + name: pvcModule.name, + config: serviceConfig, + disabled: pvcModule.disabled, + module, + sourceModule: module, + spec, + } +} diff --git a/garden-service/static/kubernetes/persistentvolumeclaim.json b/garden-service/static/kubernetes/persistentvolumeclaim.json new file mode 100644 index 0000000000..00c29c4fe2 --- /dev/null +++ b/garden-service/static/kubernetes/persistentvolumeclaim.json @@ -0,0 +1,575 @@ +{ + "description": "PersistentVolumeClaim is a user's request for and claim to a persistent volume", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": [ + "string", + "null" + ] + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": [ + "string", + "null" + ], + "enum": [ + "PersistentVolumeClaim" + ] + }, + "metadata": { + "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", + "properties": { + "annotations": { + "additionalProperties": { + "type": [ + "string", + "null" + ] + }, + "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", + "type": [ + "object", + "null" + ] + }, + "clusterName": { + "description": "The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.", + "type": [ + "string", + "null" + ] + }, + "creationTimestamp": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "deletionGracePeriodSeconds": { + "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "deletionTimestamp": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "finalizers": { + "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order. Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.", + "items": { + "type": [ + "string", + "null" + ] + }, + "type": [ + "array", + "null" + ], + "x-kubernetes-patch-strategy": "merge" + }, + "generateName": { + "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency", + "type": [ + "string", + "null" + ] + }, + "generation": { + "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "labels": { + "additionalProperties": { + "type": [ + "string", + "null" + ] + }, + "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", + "type": [ + "object", + "null" + ] + }, + "managedFields": { + "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.", + "items": { + "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", + "properties": { + "apiVersion": { + "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", + "type": [ + "string", + "null" + ] + }, + "fieldsType": { + "description": "FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: \"FieldsV1\"", + "type": [ + "string", + "null" + ] + }, + "fieldsV1": { + "description": "FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.\n\nEach key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:', where is the name of a field in a struct, or key in a map 'v:', where is the exact json formatted value of a list item 'i:', where is position of a item in a list 'k:', where is a map of a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.\n\nThe exact format is defined in sigs.k8s.io/structured-merge-diff", + "type": [ + "object", + "null" + ] + }, + "manager": { + "description": "Manager is an identifier of the workflow managing these fields.", + "type": [ + "string", + "null" + ] + }, + "operation": { + "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", + "type": [ + "string", + "null" + ] + }, + "time": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "format": "date-time", + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "type": [ + "array", + "null" + ] + }, + "name": { + "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names", + "type": [ + "string", + "null" + ] + }, + "namespace": { + "description": "Namespace defines the space within each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces", + "type": [ + "string", + "null" + ] + }, + "ownerReferences": { + "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", + "items": { + "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", + "properties": { + "apiVersion": { + "description": "API version of the referent.", + "type": "string" + }, + "blockOwnerDeletion": { + "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", + "type": [ + "boolean", + "null" + ] + }, + "controller": { + "description": "If true, this reference points to the managing controller.", + "type": [ + "boolean", + "null" + ] + }, + "kind": { + "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "name": { + "description": "Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names", + "type": "string" + }, + "uid": { + "description": "UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", + "type": "string" + } + }, + "required": [ + "apiVersion", + "kind", + "name", + "uid" + ], + "type": [ + "object", + "null" + ] + }, + "type": [ + "array", + "null" + ], + "x-kubernetes-patch-merge-key": "uid", + "x-kubernetes-patch-strategy": "merge" + }, + "resourceVersion": { + "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency", + "type": [ + "string", + "null" + ] + }, + "selfLink": { + "description": "SelfLink is a URL representing this object. Populated by the system. Read-only.\n\nDEPRECATED Kubernetes will stop propagating this field in 1.20 release and the field is planned to be removed in 1.21 release.", + "type": [ + "string", + "null" + ] + }, + "uid": { + "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids", + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "spec": { + "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", + "properties": { + "accessModes": { + "description": "AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", + "items": { + "type": [ + "string", + "null" + ] + }, + "type": [ + "array", + "null" + ] + }, + "dataSource": { + "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", + "properties": { + "apiGroup": { + "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", + "type": [ + "string", + "null" + ] + }, + "kind": { + "description": "Kind is the type of resource being referenced", + "type": "string" + }, + "name": { + "description": "Name is the name of resource being referenced", + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "type": [ + "object", + "null" + ] + }, + "resources": { + "description": "ResourceRequirements describes the compute resource requirements.", + "properties": { + "limits": { + "additionalProperties": { + "oneOf": [ + { + "type": [ + "string", + "null" + ] + }, + { + "type": [ + "number", + "null" + ] + } + ] + }, + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", + "type": [ + "object", + "null" + ] + }, + "requests": { + "additionalProperties": { + "oneOf": [ + { + "type": [ + "string", + "null" + ] + }, + { + "type": [ + "number", + "null" + ] + } + ] + }, + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", + "type": [ + "object", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "selector": { + "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", + "properties": { + "matchExpressions": { + "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", + "items": { + "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", + "properties": { + "key": { + "description": "key is the label key that the selector applies to.", + "type": "string", + "x-kubernetes-patch-merge-key": "key", + "x-kubernetes-patch-strategy": "merge" + }, + "operator": { + "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + "type": "string" + }, + "values": { + "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", + "items": { + "type": [ + "string", + "null" + ] + }, + "type": [ + "array", + "null" + ] + } + }, + "required": [ + "key", + "operator" + ], + "type": [ + "object", + "null" + ] + }, + "type": [ + "array", + "null" + ] + }, + "matchLabels": { + "additionalProperties": { + "type": [ + "string", + "null" + ] + }, + "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", + "type": [ + "object", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "storageClassName": { + "description": "Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", + "type": [ + "string", + "null" + ] + }, + "volumeMode": { + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. This is a beta feature.", + "type": [ + "string", + "null" + ] + }, + "volumeName": { + "description": "VolumeName is the binding reference to the PersistentVolume backing this claim.", + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "status": { + "description": "PersistentVolumeClaimStatus is the current status of a persistent volume claim.", + "properties": { + "accessModes": { + "description": "AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", + "items": { + "type": [ + "string", + "null" + ] + }, + "type": [ + "array", + "null" + ] + }, + "capacity": { + "additionalProperties": { + "oneOf": [ + { + "type": [ + "string", + "null" + ] + }, + { + "type": [ + "number", + "null" + ] + } + ] + }, + "description": "Represents the actual resources of the underlying volume.", + "type": [ + "object", + "null" + ] + }, + "conditions": { + "description": "Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.", + "items": { + "description": "PersistentVolumeClaimCondition contails details about state of pvc", + "properties": { + "lastProbeTime": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "lastTransitionTime": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "message": { + "description": "Human-readable message indicating details about last transition.", + "type": [ + "string", + "null" + ] + }, + "reason": { + "description": "Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.", + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "status" + ], + "type": [ + "object", + "null" + ] + }, + "type": [ + "array", + "null" + ], + "x-kubernetes-patch-merge-key": "type", + "x-kubernetes-patch-strategy": "merge" + }, + "phase": { + "description": "Phase represents the current phase of PersistentVolumeClaim.", + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + } + }, + "type": "object", + "x-kubernetes-group-version-kind": [ + { + "group": "", + "kind": "PersistentVolumeClaim", + "version": "v1" + } + ], + "$schema": "http://json-schema.org/schema#" +} \ No newline at end of file diff --git a/garden-service/test/data/test-projects/container/simple-service/Dockerfile b/garden-service/test/data/test-projects/container/simple-service/Dockerfile index 254c6c1bdb..0eb303f0c7 100644 --- a/garden-service/test/data/test-projects/container/simple-service/Dockerfile +++ b/garden-service/test/data/test-projects/container/simple-service/Dockerfile @@ -1,5 +1 @@ -FROM busybox:1.31.1 - -COPY main . - -ENTRYPOINT ./main +FROM busybox:1.31.1 \ No newline at end of file diff --git a/garden-service/test/data/test-projects/container/simple-service/garden.yml b/garden-service/test/data/test-projects/container/simple-service/garden.yml index 806953d63d..0f4dc43964 100644 --- a/garden-service/test/data/test-projects/container/simple-service/garden.yml +++ b/garden-service/test/data/test-projects/container/simple-service/garden.yml @@ -4,6 +4,7 @@ description: Test module for a simple service type: container services: - name: simple-service + command: [sh, -c, "echo Server running... && nc -l -p 8080"] ports: - name: http containerPort: 8080 diff --git a/garden-service/test/data/test-projects/container/simple-service/main b/garden-service/test/data/test-projects/container/simple-service/main deleted file mode 100755 index 9639408e7a..0000000000 Binary files a/garden-service/test/data/test-projects/container/simple-service/main and /dev/null differ diff --git a/garden-service/test/data/test-projects/container/simple-service/main.go b/garden-service/test/data/test-projects/container/simple-service/main.go deleted file mode 100644 index 34ae7d9838..0000000000 --- a/garden-service/test/data/test-projects/container/simple-service/main.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "fmt" - "net/http" -) - -func handler(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, "Hello from Go!") -} - -func main() { - http.HandleFunc("/hello-backend", handler) - fmt.Println("Server running...") - - http.ListenAndServe(":8080", nil) -} diff --git a/garden-service/test/data/test-projects/container/volume-reference/garden.yml b/garden-service/test/data/test-projects/container/volume-reference/garden.yml new file mode 100644 index 0000000000..d1f0310ac7 --- /dev/null +++ b/garden-service/test/data/test-projects/container/volume-reference/garden.yml @@ -0,0 +1,29 @@ +kind: Module +name: volume-reference +description: Test module for volume module references +type: container +image: busybox:1.31.1 +include: [] +build: + dependencies: [simple-service] +services: + - name: volume-reference + command: [sh, -c, "nc -l -p 8080"] + ports: + - name: http + containerPort: 8080 + volumes: + - name: test + module: volume-module + containerPath: /volume +--- +kind: Module +name: volume-module +type: persistentvolumeclaim +include: [] +# Note: This is never actually deployed +spec: + accessModes: [ReadOnlyMany] + resources: + requests: + storage: 10Mi \ No newline at end of file diff --git a/garden-service/test/data/test-projects/persistentvolumeclaim/garden.yml b/garden-service/test/data/test-projects/persistentvolumeclaim/garden.yml new file mode 100644 index 0000000000..b89758d097 --- /dev/null +++ b/garden-service/test/data/test-projects/persistentvolumeclaim/garden.yml @@ -0,0 +1,25 @@ +kind: Project +name: persistentvolumeclaim +providers: + - name: local-kubernetes +--- +kind: Module +name: volume-module +type: persistentvolumeclaim +spec: + accessModes: [ReadWriteOnce] +--- +kind: Module +name: simple-service +type: container +image: busybox:1.31.1 +services: + - name: simple-service + command: [sh, -c, "touch /volume/foo.txt && nc -l -p 8080"] + ports: + - name: http + containerPort: 8080 + volumes: + - name: test + containerPath: /volume + module: volume-module diff --git a/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts b/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts index b87396be13..62cf3cf41b 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts @@ -19,6 +19,8 @@ import { cloneDeep, keyBy } from "lodash" import { getContainerTestGarden } from "./container" import { DeployTask } from "../../../../../../src/tasks/deploy" import { getServiceStatuses } from "../../../../../../src/tasks/base" +import { expectError } from "../../../../../helpers" +import stripAnsi = require("strip-ansi") describe("kubernetes container deployment handlers", () => { let garden: Garden @@ -26,6 +28,10 @@ describe("kubernetes container deployment handlers", () => { let provider: KubernetesProvider let api: KubeApi + beforeEach(async () => { + graph = await garden.getConfigGraph(garden.log) + }) + after(async () => { if (garden) { await garden.close() @@ -34,8 +40,6 @@ describe("kubernetes container deployment handlers", () => { const init = async (environmentName: string) => { garden = await getContainerTestGarden(environmentName) - - graph = await garden.getConfigGraph(garden.log) provider = await garden.resolveProvider("local-kubernetes") api = await KubeApi.factory(garden.log, provider) } @@ -81,6 +85,7 @@ describe("kubernetes container deployment handlers", () => { { name: "simple-service", image: "simple-service:" + version, + command: ["sh", "-c", "echo Server running... && nc -l -p 8080"], env: [ { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" } } }, { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } }, @@ -142,6 +147,52 @@ describe("kubernetes container deployment handlers", () => { expect(copiedSecret).to.exist expect(resource.spec.template.spec.imagePullSecrets).to.eql([{ name: secretName }]) }) + + it("should correctly mount a referenced PVC module", async () => { + const service = await graph.getService("volume-reference") + const namespace = garden.projectName + + const resource = await createWorkloadManifest({ + api, + provider, + service, + runtimeContext: emptyRuntimeContext, + namespace, + enableHotReload: false, + log: garden.log, + production: false, + }) + + expect(resource.spec.template.spec.volumes).to.eql([ + { name: "test", persistentVolumeClaim: { claimName: "volume-module" } }, + ]) + expect(resource.spec.template.spec.containers[0].volumeMounts).to.eql([{ name: "test", mountPath: "/volume" }]) + }) + + it("should throw if incompatible module is specified as a volume module", async () => { + const service = await graph.getService("volume-reference") + const namespace = garden.projectName + + service.spec.volumes = [{ name: "test", module: "simple-service" }] + + await expectError( + () => + createWorkloadManifest({ + api, + provider, + service, + runtimeContext: emptyRuntimeContext, + namespace, + enableHotReload: false, + log: garden.log, + production: false, + }), + (err) => + expect(stripAnsi(err.message)).to.equal( + "Container module volume-reference specifies a unsupported module simple-service for volume mount test. Only persistentvolumeclaim modules are supported at this time." + ) + ) + }) }) describe("deployContainerService", () => { @@ -170,6 +221,32 @@ describe("kubernetes container deployment handlers", () => { `${service.name}:${service.module.version.versionString}` ) }) + + it("should deploy a service referencing a volume module", async () => { + const service = await graph.getService("volume-reference") + + const deployTask = new DeployTask({ + garden, + graph, + log: garden.log, + service, + force: true, + forceBuild: false, + }) + + const results = await garden.processTasks([deployTask], { throwOnError: true }) + const statuses = getServiceStatuses(results) + const status = statuses[service.name] + const resources = keyBy(status.detail["remoteResources"], "kind") + + expect(status.state === "ready") + expect(resources.Deployment.spec.template.spec.volumes).to.eql([ + { name: "test", persistentVolumeClaim: { claimName: "volume-module" } }, + ]) + expect(resources.Deployment.spec.template.spec.containers[0].volumeMounts).to.eql([ + { name: "test", mountPath: "/volume" }, + ]) + }) }) context("cluster-docker mode", () => { @@ -197,6 +274,32 @@ describe("kubernetes container deployment handlers", () => { `127.0.0.1:5000/container/${service.name}:${service.module.version.versionString}` ) }) + + it("should deploy a service referencing a volume module", async () => { + const service = await graph.getService("volume-reference") + + const deployTask = new DeployTask({ + garden, + graph, + log: garden.log, + service, + force: true, + forceBuild: false, + }) + + const results = await garden.processTasks([deployTask], { throwOnError: true }) + const statuses = getServiceStatuses(results) + const status = statuses[service.name] + const resources = keyBy(status.detail["remoteResources"], "kind") + + expect(status.state === "ready") + expect(resources.Deployment.spec.template.spec.volumes).to.eql([ + { name: "test", persistentVolumeClaim: { claimName: "volume-module" } }, + ]) + expect(resources.Deployment.spec.template.spec.containers[0].volumeMounts).to.eql([ + { name: "test", mountPath: "/volume" }, + ]) + }) }) context("kaniko mode", () => { diff --git a/garden-service/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts b/garden-service/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts new file mode 100644 index 0000000000..67a3b55908 --- /dev/null +++ b/garden-service/test/integ/src/plugins/kubernetes/volume/persistentvolumeclaim.ts @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import tmp from "tmp-promise" +import { ProjectConfig } from "../../../../../../src/config/project" +import execa = require("execa") +import { DEFAULT_API_VERSION } from "../../../../../../src/constants" +import { expect } from "chai" +import { TestGarden, makeTempDir } from "../../../../../helpers" +import { DeployTask } from "../../../../../../src/tasks/deploy" +import { emptyRuntimeContext } from "../../../../../../src/runtime-context" +import { isSubset } from "../../../../../../src/util/is-subset" + +describe("persistentvolumeclaim", () => { + let tmpDir: tmp.DirectoryResult + let projectConfigFoo: ProjectConfig + + before(async () => { + tmpDir = await makeTempDir() + + await execa("git", ["init"], { cwd: tmpDir.path }) + + projectConfigFoo = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: tmpDir.path, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", variables: {} }], + providers: [{ name: "local-kubernetes", namespace: "default" }], + variables: {}, + } + }) + + after(async () => { + await tmpDir.cleanup() + }) + + it("should successfully deploy a simple PVC", async () => { + const garden = await TestGarden.factory(tmpDir.path, { + plugins: [], + config: projectConfigFoo, + }) + + const spec = { + accessModes: ["ReadOnlyMany"], + resources: { + requests: { + storage: "10Mi", + }, + }, + } + + garden.setModuleConfigs([ + { + apiVersion: DEFAULT_API_VERSION, + name: "test", + type: "persistentvolumeclaim", + allowPublish: false, + build: { dependencies: [] }, + disabled: false, + outputs: {}, + path: tmpDir.path, + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + spec: { + spec, + }, + }, + ]) + + const graph = await garden.getConfigGraph(garden.log) + const service = await graph.getService("test") + + const deployTask = new DeployTask({ + garden, + graph, + log: garden.log, + service, + force: true, + forceBuild: false, + }) + + await garden.processTasks([deployTask], { throwOnError: true }) + + const actions = await garden.getActionRouter() + const status = await actions.getServiceStatus({ + log: garden.log, + service, + hotReload: false, + runtimeContext: emptyRuntimeContext, + }) + + const remoteResources = status.detail["remoteResources"] + + expect(status.state === "ready") + expect(remoteResources.length).to.equal(1) + expect( + isSubset(remoteResources[0], { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { name: "test", namespace: "default" }, + spec, + }) + ).to.be.true + + await actions.deleteService({ log: garden.log, service }) + }) +}) diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index 29b418fe56..c8bfbc2605 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -205,6 +205,39 @@ describe("ActionRouter", () => { }) describe("module actions", () => { + describe("configureModule", () => { + it("should consolidate the declared build dependencies", async () => { + const moduleConfigA = (await garden.getRawModuleConfigs(["module-a"]))[0] + + const moduleConfig = { + ...moduleConfigA, + build: { + dependencies: [ + { name: "module-b", copy: [{ source: "1", target: "1" }] }, + { name: "module-b", copy: [{ source: "2", target: "2" }] }, + { name: "module-b", copy: [{ source: "2", target: "2" }] }, + { name: "module-c", copy: [{ source: "3", target: "3" }] }, + ], + }, + } + + const result = await actions.configureModule({ log, moduleConfig }) + expect(result.moduleConfig.build.dependencies).to.eql([ + { + name: "module-b", + copy: [ + { source: "1", target: "1" }, + { source: "2", target: "2" }, + ], + }, + { + name: "module-c", + copy: [{ source: "3", target: "3" }], + }, + ]) + }) + }) + describe("getBuildStatus", () => { it("should correctly call the corresponding plugin handler", async () => { const result = await actions.getBuildStatus({ log, module }) diff --git a/garden-service/test/unit/src/commands/get/get-config.ts b/garden-service/test/unit/src/commands/get/get-config.ts index efd26559fd..b4b250dbc6 100644 --- a/garden-service/test/unit/src/commands/get/get-config.ts +++ b/garden-service/test/unit/src/commands/get/get-config.ts @@ -292,6 +292,7 @@ describe("GetConfigCommand", () => { disabled: false, timeout: null, env: {}, + volumes: [], }, timeout: null, }, @@ -305,7 +306,7 @@ describe("GetConfigCommand", () => { projectRoot: garden.projectRoot, } - expect(config).to.deep.equal(res.result) + expect(res.result).to.deep.equal(config) }) it("should exclude disabled test configs", async () => { @@ -383,6 +384,7 @@ describe("GetConfigCommand", () => { disabled: false, timeout: null, env: {}, + volumes: [], }, timeout: null, }, @@ -396,6 +398,6 @@ describe("GetConfigCommand", () => { projectRoot: garden.projectRoot, } - expect(config).to.deep.equal(res.result) + expect(res.result).to.deep.equal(config) }) }) diff --git a/garden-service/test/unit/src/config/common.ts b/garden-service/test/unit/src/config/common.ts index 8e21fec5a1..1230292ed2 100644 --- a/garden-service/test/unit/src/config/common.ts +++ b/garden-service/test/unit/src/config/common.ts @@ -348,6 +348,56 @@ describe("joiRepositoryUrl", () => { }) }) +describe("joi.customObject", () => { + const jsonSchema = { + type: "object", + properties: { + stringProperty: { type: "string" }, + numberProperty: { type: "integer", default: 999 }, + }, + additionalProperties: false, + required: ["stringProperty"], + } + + it("should validate an object with a JSON Schema", () => { + const joiSchema = joi.customObject().jsonSchema(jsonSchema) + const value = { stringProperty: "foo", numberProperty: 123 } + const result = validateSchema(value, joiSchema) + expect(result).to.eql({ stringProperty: "foo", numberProperty: 123 }) + }) + + it("should apply default values based on the JSON Schema", () => { + const joiSchema = joi.customObject().jsonSchema(jsonSchema) + const result = validateSchema({ stringProperty: "foo" }, joiSchema) + expect(result).to.eql({ stringProperty: "foo", numberProperty: 999 }) + }) + + it("should give validation error if object doesn't match specified JSON Schema", async () => { + const joiSchema = joi.customObject().jsonSchema(jsonSchema) + await expectError( + () => validateSchema({ numberProperty: "oops", blarg: "blorg" }, joiSchema), + (err) => + expect(stripAnsi(err.message)).to.equal( + "Validation error: value at . should not have additional properties, value at . should have required property 'stringProperty', value at ..numberProperty should be integer" + ) + ) + }) + + it("should throw if schema with wrong type is passed to .jsonSchema()", async () => { + await expectError( + () => joi.customObject().jsonSchema({ type: "number" }), + (err) => expect(err.message).to.equal("jsonSchema must be a valid JSON Schema with type=object or reference") + ) + }) + + it("should throw if invalid schema is passed to .jsonSchema()", async () => { + await expectError( + () => joi.customObject().jsonSchema({ type: "banana", blorg: "blarg" }), + (err) => expect(err.message).to.equal("jsonSchema must be a valid JSON Schema with type=object or reference") + ) + }) +}) + describe("validateSchema", () => { it("should format a basic object validation error", async () => { const schema = joi.object().keys({ foo: joi.string() }) diff --git a/garden-service/test/unit/src/docs/config.ts b/garden-service/test/unit/src/docs/config.ts index 7d35373f11..eb9caa5cdc 100644 --- a/garden-service/test/unit/src/docs/config.ts +++ b/garden-service/test/unit/src/docs/config.ts @@ -8,31 +8,24 @@ import { renderSchemaDescriptionYaml, - getDefaultValue, - normalizeSchemaDescriptions, renderConfigReference, - NormalizedSchemaDescription, renderMarkdownLink, sanitizeYamlStringForGitBook, - Description, } from "../../../../src/docs/config" import { expect } from "chai" -import dedent = require("dedent") import { joiArray, joi, joiEnvVars } from "../../../../src/config/common" import { buildDependencySchema } from "../../../../src/config/module" +import { normalizeJoiSchemaDescription, JoiDescription } from "../../../../src/docs/joi-schema" +import { NormalizedSchemaDescription } from "../../../../src/docs/common" +import { dedent } from "../../../../src/util/string" -describe("config", () => { +describe("docs config module", () => { const servicePortSchema = joi .number() .default((parent) => (parent ? parent.containerPort : undefined)) .example("8080") .description("description") - const testDefaultSchema = joi - .number() - .default(() => "result") - .description("description") - const testObject = joi .object() .keys({ @@ -118,7 +111,7 @@ describe("config", () => { describe("renderSchemaDescriptionYaml", () => { it("should render the yaml with the full description", () => { - const schemaDescriptions = normalizeSchemaDescriptions(portSchema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(portSchema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { renderRequired: true }) expect(yaml).to.equal(dedent` # description @@ -168,7 +161,7 @@ describe("config", () => { `) }) it("should optionally render the yaml with a basic description", () => { - const schemaDescriptions = normalizeSchemaDescriptions(portSchema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(portSchema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { renderBasicDescription: true }) expect(yaml).to.equal(dedent` # description @@ -190,7 +183,7 @@ describe("config", () => { `) }) it("should optionally skip the commented description above the key", () => { - const schemaDescriptions = normalizeSchemaDescriptions(portSchema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(portSchema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { renderFullDescription: false }) expect(yaml).to.equal(dedent` containerPort: @@ -202,7 +195,7 @@ describe("config", () => { `) }) it("should conditionally print ellipsis between object keys", () => { - const schemaDescriptions = normalizeSchemaDescriptions(portSchema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(portSchema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { renderFullDescription: false, renderEllipsisBetweenKeys: true, @@ -226,7 +219,7 @@ describe("config", () => { boo: "far", }), }) - const schemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { renderFullDescription: false, renderEllipsisBetweenKeys: true, @@ -250,7 +243,7 @@ describe("config", () => { .default(() => ({ dependencies: [] })) .description("Specify how to build the module. Note that plugins may define additional keys on this object.") - const schemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { renderFullDescription: false, renderValue: "default", @@ -270,7 +263,7 @@ describe("config", () => { dependencies: joi.string().description("Check out [some link](http://example.com)."), }) - const schemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { filterMarkdown: true, renderBasicDescription: true, @@ -288,7 +281,7 @@ describe("config", () => { dependencies: joi.string().description("Check out [some link](http://example.com)."), }) - const schemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { filterMarkdown: true, renderBasicDescription: true, @@ -308,7 +301,7 @@ describe("config", () => { keyC: joi.string(), }) - const schemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { filterMarkdown: true, presetValues: { keyC: "foo" }, @@ -331,7 +324,7 @@ describe("config", () => { keyC: joi.string(), }) - const schemaDescriptions = normalizeSchemaDescriptions(schema.describe() as Description) + const schemaDescriptions = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) const yaml = renderSchemaDescriptionYaml(schemaDescriptions, { commentOutEmpty: true, filterMarkdown: true, @@ -349,13 +342,6 @@ describe("config", () => { }) }) - describe("getDefaultValue", () => { - it("should get the default return of the function over the param", () => { - const value = getDefaultValue(testDefaultSchema.describe() as Description) - expect(value).to.eq("result") - }) - }) - describe("renderConfigReference", () => { it("should return the correct markdown", () => { const { markdownReference } = renderConfigReference(portSchema) @@ -444,59 +430,63 @@ describe("config", () => { describe("renderMarkdownLink", () => { it("should return a markdown link with a name and relative path", () => { - const happy: NormalizedSchemaDescription = { - name: "happy", + const common = { + allowedValuesOnly: false, + deprecated: false, + experimental: false, + internal: false, level: 0, required: false, + } + + const happy: NormalizedSchemaDescription = { + ...common, + type: "string", + name: "happy", hasChildren: true, fullKey: "happy", formattedName: "happy", formattedType: "string", - deprecated: false, } const families: NormalizedSchemaDescription = { + ...common, + type: "array", name: "families", - level: 0, - required: false, - hasChildren: true, fullKey: "happy.families[]", formattedName: "families[]", formattedType: "array", + hasChildren: true, parent: happy, - deprecated: false, } const are: NormalizedSchemaDescription = { + ...common, + type: "string", name: "happy", - level: 0, - required: false, - hasChildren: true, fullKey: "happy.families[].are", formattedName: "are", formattedType: "string", + hasChildren: true, parent: families, - deprecated: false, } const all: NormalizedSchemaDescription = { + ...common, + type: "array", name: "all", - level: 0, - required: false, - hasChildren: true, fullKey: "happy.families[].are.all[]", formattedName: "all[]", formattedType: "array", + hasChildren: true, parent: are, - deprecated: false, } const alike: NormalizedSchemaDescription = { + ...common, + type: "string", name: "alike", - level: 0, - required: false, - hasChildren: false, - fullKey: "happy.families[].are.all[].alike", formattedName: "alike", formattedType: "string", + fullKey: "happy.families[].are.all[].alike", + hasChildren: false, parent: all, - deprecated: false, } expect(renderMarkdownLink(alike)).to.equal(`[alike](#happyfamiliesareallalike)`) diff --git a/garden-service/test/unit/src/docs/joi-schema.ts b/garden-service/test/unit/src/docs/joi-schema.ts new file mode 100644 index 0000000000..5507b6c272 --- /dev/null +++ b/garden-service/test/unit/src/docs/joi-schema.ts @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { expect } from "chai" +import { joi } from "../../../../src/config/common" +import { getJoiDefaultValue, JoiDescription, normalizeJoiSchemaDescription } from "../../../../src/docs/joi-schema" +import { testJsonSchema } from "./json-schema" +import { normalizeJsonSchema } from "../../../../src/docs/json-schema" + +describe("normalizeJoiSchemaDescription", () => { + it("should correctly handle joi.customObject().jsonSchema() schemas", async () => { + const schema = joi.customObject().jsonSchema(testJsonSchema) + const result = normalizeJoiSchemaDescription(schema.describe() as JoiDescription) + expect(result).to.eql(normalizeJsonSchema(testJsonSchema)) + }) +}) + +describe("getJoiDefaultValue", () => { + const testDefaultSchema = joi + .number() + .default(() => "result") + .description("description") + + it("should get the default return of the function over the param", () => { + const value = getJoiDefaultValue(testDefaultSchema.describe() as JoiDescription) + expect(value).to.equal("result") + }) +}) diff --git a/garden-service/test/unit/src/docs/json-schema.ts b/garden-service/test/unit/src/docs/json-schema.ts new file mode 100644 index 0000000000..181d26d097 --- /dev/null +++ b/garden-service/test/unit/src/docs/json-schema.ts @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { expect } from "chai" +import { normalizeJsonSchema } from "../../../../src/docs/json-schema" + +describe("normalizeJsonSchema", () => { + it("should normalize a type=oject JSON Schema", () => { + const keys = normalizeJsonSchema(testJsonSchema) + + expect(keys).to.eql([ + { + type: "string", + name: "apiVersion", + allowedValuesOnly: false, + defaultValue: "v1", + deprecated: false, + description: testJsonSchema.properties.apiVersion.description, + experimental: false, + fullKey: "apiVersion", + formattedExample: undefined, + formattedName: "apiVersion", + formattedType: "string", + hasChildren: false, + internal: false, + level: 0, + parent: undefined, + required: false, + }, + { + type: "string", + name: "kind", + allowedValuesOnly: true, + defaultValue: undefined, + deprecated: false, + description: testJsonSchema.properties.kind.description, + experimental: false, + fullKey: "kind", + formattedExample: undefined, + formattedName: "kind", + formattedType: "string", + hasChildren: false, + internal: false, + level: 0, + parent: undefined, + required: false, + allowedValues: '"PersistentVolumeClaim"', + }, + { + type: "object", + name: "metadata", + allowedValuesOnly: false, + defaultValue: undefined, + deprecated: false, + description: testJsonSchema.properties.metadata.description, + experimental: false, + fullKey: "metadata", + formattedExample: undefined, + formattedName: "metadata", + formattedType: "object", + hasChildren: true, + internal: false, + level: 0, + parent: undefined, + required: false, + }, + { + type: "string", + name: "lastTransitionTime", + allowedValuesOnly: false, + defaultValue: undefined, + deprecated: false, + description: testJsonSchema.properties.metadata.properties.lastTransitionTime.description, + experimental: false, + fullKey: "metadata.lastTransitionTime", + formattedExample: undefined, + formattedName: "lastTransitionTime", + formattedType: "string", + hasChildren: false, + internal: false, + level: 1, + parent: { + type: "object", + name: "metadata", + allowedValuesOnly: false, + defaultValue: undefined, + deprecated: false, + description: testJsonSchema.properties.metadata.description, + experimental: false, + fullKey: "metadata", + formattedExample: undefined, + formattedName: "metadata", + formattedType: "object", + hasChildren: true, + internal: false, + level: 0, + parent: undefined, + required: false, + }, + required: false, + }, + ]) + }) +}) + +export const testJsonSchema = { + description: "PersistentVolumeClaim is a user's request for and claim to a persistent volume", + properties: { + apiVersion: { + description: + "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + type: ["string", "null"], + default: "v1", + }, + kind: { + description: + "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + type: ["string", "null"], + enum: ["PersistentVolumeClaim"], + }, + metadata: { + description: + "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", + properties: { + lastTransitionTime: { + description: + "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + format: "date-time", + type: ["string", "null"], + example: "2020-01-01T00:00:00", + }, + }, + type: ["object", "null"], + }, + }, + type: "object", + $schema: "http://json-schema.org/schema#", +} diff --git a/garden-service/test/unit/src/docs/util.ts b/garden-service/test/unit/src/docs/util.ts index 8aebf9a1af..0113de597f 100644 --- a/garden-service/test/unit/src/docs/util.ts +++ b/garden-service/test/unit/src/docs/util.ts @@ -8,7 +8,7 @@ import { dedent } from "../../../../src/util/string" import { expect } from "chai" -import { convertMarkdownLinks } from "../../../../src/docs/util" +import { convertMarkdownLinks } from "../../../../src/docs/common" describe("convertMarkdownLinks", () => { it("should convert all markdown links in the given text to plain links", () => { diff --git a/garden-service/test/unit/src/garden.ts b/garden-service/test/unit/src/garden.ts index efd92a0046..7db87270d0 100644 --- a/garden-service/test/unit/src/garden.ts +++ b/garden-service/test/unit/src/garden.ts @@ -3149,9 +3149,9 @@ describe("Garden", () => { }) context("test against fixed version hashes", async () => { - const moduleAVersionString = "v-0ef068b6a4" - const moduleBVersionString = "v-53b030b72a" - const moduleCVersionString = "v-a362160029" + const moduleAVersionString = "v-4b68c1fda7" + const moduleBVersionString = "v-e145423c6c" + const moduleCVersionString = "v-73c52d0676" it("should return the same module versions between runtimes", async () => { const projectRoot = getDataDir("test-projects", "fixed-version-hashes-1") diff --git a/garden-service/test/unit/src/plugins/container/container.ts b/garden-service/test/unit/src/plugins/container/container.ts index 98f59795b4..71983258e0 100644 --- a/garden-service/test/unit/src/plugins/container/container.ts +++ b/garden-service/test/unit/src/plugins/container/container.ts @@ -95,7 +95,7 @@ describe("plugins.container", () => { return moduleFromConfig(garden, graph, parsed.moduleConfig) } - describe("validate", () => { + describe("configureContainerModule", () => { it("should validate and parse a container module", async () => { const moduleConfig: ContainerModuleConfig = { allowPublish: false, @@ -168,6 +168,7 @@ describe("plugins.container", () => { TASK_ENV_VAR: "value", }, timeout: null, + volumes: [], }, ], tests: [ @@ -181,6 +182,7 @@ describe("plugins.container", () => { TEST_ENV_VAR: "value", }, timeout: null, + volumes: [], }, ], }, @@ -253,6 +255,7 @@ describe("plugins.container", () => { TASK_ENV_VAR: "value", }, timeout: null, + volumes: [], }, ], tests: [ @@ -266,6 +269,7 @@ describe("plugins.container", () => { TEST_ENV_VAR: "value", }, timeout: null, + volumes: [], }, ], }, @@ -320,6 +324,7 @@ describe("plugins.container", () => { }, name: "task-a", timeout: null, + volumes: [], }, timeout: null, }, @@ -339,6 +344,7 @@ describe("plugins.container", () => { TEST_ENV_VAR: "value", }, timeout: null, + volumes: [], }, timeout: null, }, @@ -347,6 +353,174 @@ describe("plugins.container", () => { }) }) + it("should add service volume modules as build and runtime dependencies", async () => { + const moduleConfig: ContainerModuleConfig = { + allowPublish: false, + build: { + dependencies: [], + }, + disabled: false, + apiVersion: "garden.io/v0", + name: "module-a", + outputs: {}, + path: modulePath, + type: "container", + + spec: { + build: { + dependencies: [], + timeout: DEFAULT_BUILD_TIMEOUT, + }, + buildArgs: {}, + extraFlags: [], + services: [ + { + name: "service-a", + annotations: {}, + args: ["echo"], + dependencies: [], + daemon: false, + disabled: false, + ingresses: [], + env: {}, + healthCheck: {}, + limits: { + cpu: 123, + memory: 456, + }, + ports: [], + replicas: 1, + volumes: [ + { + name: "test", + containerPath: "/", + module: "volume-module", + }, + ], + }, + ], + tasks: [], + tests: [], + }, + + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + } + + const result = await configure({ ctx, moduleConfig, log }) + + expect(result.moduleConfig.build.dependencies).to.eql([{ name: "volume-module", copy: [] }]) + expect(result.moduleConfig.serviceConfigs[0].dependencies).to.eql(["volume-module"]) + }) + + it("should add task volume modules as build and runtime dependencies", async () => { + const moduleConfig: ContainerModuleConfig = { + allowPublish: false, + build: { + dependencies: [], + }, + disabled: false, + apiVersion: "garden.io/v0", + name: "module-a", + outputs: {}, + path: modulePath, + type: "container", + + spec: { + build: { + dependencies: [], + timeout: DEFAULT_BUILD_TIMEOUT, + }, + buildArgs: {}, + extraFlags: [], + services: [], + tasks: [ + { + name: "task-a", + args: [], + artifacts: [], + cacheResult: true, + dependencies: [], + disabled: false, + env: {}, + timeout: null, + volumes: [ + { + name: "test", + containerPath: "/", + module: "volume-module", + }, + ], + }, + ], + tests: [], + }, + + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + } + + const result = await configure({ ctx, moduleConfig, log }) + + expect(result.moduleConfig.build.dependencies).to.eql([{ name: "volume-module", copy: [] }]) + expect(result.moduleConfig.taskConfigs[0].dependencies).to.eql(["volume-module"]) + }) + + it("should add test volume modules as build and runtime dependencies", async () => { + const moduleConfig: ContainerModuleConfig = { + allowPublish: false, + build: { + dependencies: [], + }, + disabled: false, + apiVersion: "garden.io/v0", + name: "module-a", + outputs: {}, + path: modulePath, + type: "container", + + spec: { + build: { + dependencies: [], + timeout: DEFAULT_BUILD_TIMEOUT, + }, + buildArgs: {}, + extraFlags: [], + services: [], + tasks: [], + tests: [ + { + name: "test-a", + args: [], + artifacts: [], + dependencies: [], + disabled: false, + env: {}, + timeout: null, + volumes: [ + { + name: "test", + containerPath: "/", + module: "volume-module", + }, + ], + }, + ], + }, + + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + } + + const result = await configure({ ctx, moduleConfig, log }) + + expect(result.moduleConfig.build.dependencies).to.eql([{ name: "volume-module", copy: [] }]) + expect(result.moduleConfig.testConfigs[0].dependencies).to.eql(["volume-module"]) + }) + it("should fail with invalid port in ingress spec", async () => { const moduleConfig: ContainerModuleConfig = { allowPublish: false, @@ -399,6 +573,7 @@ describe("plugins.container", () => { disabled: false, env: {}, timeout: null, + volumes: [], }, ], tests: [ @@ -410,6 +585,7 @@ describe("plugins.container", () => { disabled: false, env: {}, timeout: null, + volumes: [], }, ], }, @@ -474,6 +650,7 @@ describe("plugins.container", () => { disabled: false, env: {}, timeout: null, + volumes: [], }, ], tests: [], @@ -536,6 +713,7 @@ describe("plugins.container", () => { disabled: false, env: {}, timeout: null, + volumes: [], }, ], tests: [], diff --git a/garden-service/test/unit/src/vcs/vcs.ts b/garden-service/test/unit/src/vcs/vcs.ts index 184acf0bbe..3442fcef45 100644 --- a/garden-service/test/unit/src/vcs/vcs.ts +++ b/garden-service/test/unit/src/vcs/vcs.ts @@ -285,7 +285,7 @@ describe("VcsHandler", () => { const garden = await makeTestGarden(projectRoot) const config = await garden.resolveModuleConfig(garden.log, "module-a") - const fixedVersionString = "v-64e51cdd17" + const fixedVersionString = "v-e0322ab204" expect(getVersionString(config, [namedVersionA, namedVersionB, namedVersionC])).to.eql(fixedVersionString) delete process.env.TEST_ENV_VAR