Docs and README update, Profiles and ES charts to pass chart linting, Update google/oauth2 version, Add GCP creds to Profile

docs/functions.rst

@@ -351,7 +351,7 @@ Example:
     args:
       namespace: "{{ .Deployment.Namespace }}"
       pod: "{{ index .Deployment.Pods 0 }}"
-      image: kanisterio/kanister-tools:0.15.0
+      image: kanisterio/kanister-tools:0.16.0
       backupArtifactPrefix: s3-bucket/path/artifactPrefix
       backupIdentifier: "{{ .Time }}"
 

docs/helm.rst

@@ -26,7 +26,7 @@ You also need to install the Kanister controller
 
 .. code-block:: console
 
-   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.15.0
+   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.16.0
 
 Kanister-Enabled Applications
 =============================

docs/helm_instructions/create_profile.rst

@@ -10,20 +10,20 @@ create as part of the first instance.
 .. note:: Prior to creating the Profile CR, you will need to do the following:
 
    * Create a bucket for artifacts on your S3 store. This will be your
-     ``s3.bucket`` parameter to the command.
-   * Obtain ``s3.accessKey`` and ``s3.secretKey`` credentials for an
+     ``location.bucket`` parameter to the command.
+   * Obtain ``aws.accessKey`` and ``aws.secretKey`` credentials for an
      account with access to the bucket that you will use.
    * Configure the permissions on the bucket to allow the account to
      list, put, get, and delete.
    * Make sure that your retention policy allows deletions so that artifacts
      can be reclaimed based on your intended data backup retention.
 
-.. note:: The ``s3.endpoint`` parameter is only required if you are using an
+.. note:: The ``location.endpoint`` parameter is only required if you are using an
    S3-compatible provider different from AWS.
 
    If you are using an on-premises s3 provider, the endpoint specified needs be
    accessible from within your Kubernetes cluster.
 
    If, in your environment, the endpoint has a self-signed SSL certificate,
-   include ``--set s3.verifySSL=false`` in the above command to disable SSL
+   include ``--set verifySSL=false`` in the above command to disable SSL
    verification for the S3 operations in the blueprint.

docs/helm_instructions/elasticsearch_instructions.rst

@@ -24,10 +24,11 @@ Then install the sample Elasticsearch application in its own namespace.
           --namespace es-test \
           --set profile.create='true' \
           --set profile.profileName='es-test-profile' \
-          --set profile.s3.endpoint='https://my-custom-s3-provider:9000' \
-          --set profile.s3.accessKey='AKIAIOSFODNN7EXAMPLE' \
-          --set profile.s3.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
-          --set profile.s3.bucket='kanister-bucket'
+          --set profile.location.type='s3Compliant' \
+          --set profile.location.bucket='kanister-bucket' \
+          --set profile.location.endpoint='https://my-custom-s3-provider:9000' \
+          --set profile.aws.accessKey='AKIAIOSFODNN7EXAMPLE' \
+          --set profile.aws.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
           --set kanister.controller_namespace="kanister"
 
 .. only:: defaultns

docs/helm_instructions/mongodb_instructions.rst

@@ -23,10 +23,11 @@ Then install the sample MongoDB replica set application in its own namespace.
           --namespace mongodb-test \
           --set profile.create='true' \
           --set profile.profileName='mongo-test-profile' \
-          --set profile.s3.bucket="kanister-bucket" \
-          --set profile.s3.endpoint="https://my-custom-s3-provider:9000" \
-          --set profile.s3.accessKey="AKIAIOSFODNN7EXAMPLE" \
-          --set profile.s3.secretKey="wJalrXUtnFEMI!K7MDENG!bPxRfiCYEXAMPLEKEY" \
+          --set profile.location.type='s3Compliant' \
+          --set profile.location.bucket='kanister-bucket' \
+          --set profile.location.endpoint='https://my-custom-s3-provider:9000' \
+          --set profile.aws.accessKey='AKIAIOSFODNN7EXAMPLE' \
+          --set profile.aws.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
           --set kanister.controller_namespace="kanister" \
           --set replicas=1 \
           --set persistentVolume.size=2Gi

docs/helm_instructions/mysql_instructions.rst

@@ -22,10 +22,11 @@ Then install the sample MySQL application in its own namespace.
      $ helm install kanister/kanister-mysql -n mysql --namespace mysql-test \
           --set profile.create='true' \
           --set profile.profileName='mysql-test-profile' \
-          --set profile.s3.bucket="kanister-bucket" \
-          --set profile.s3.endpoint="https://my-custom-s3-provider:9000" \
-          --set profile.s3.accessKey="AKIAIOSFODNN7EXAMPLE" \
-          --set profile.s3.secretKey="wJalrXUtnFEMI!K7MDENG!bPxRfiCYEXAMPLEKEY" \
+          --set profile.location.type='s3Compliant' \
+          --set profile.location.bucket='kanister-bucket' \
+          --set profile.location.endpoint='https://my-custom-s3-provider:9000' \
+          --set profile.aws.accessKey='AKIAIOSFODNN7EXAMPLE' \
+          --set profile.aws.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
           --set kanister.controller_namespace="kanister" \
           --set mysqlRootPassword="asd#45@mysqlEXAMPLE" \
           --set persistence.size=10Gi

docs/helm_instructions/pgsql_instructions.rst

@@ -26,10 +26,11 @@ Then install the sample PostgreSQL application in its own namespace.
           --namespace postgresql-test \
           --set profile.create='true' \
           --set profile.profileName='postgres-test-profile' \
-          --set profile.s3.bucket="kanister-bucket" \
-          --set profile.s3.endpoint="https://my-custom-s3-provider:9000" \
-          --set profile.s3.accessKey="AKIAIOSFODNN7EXAMPLE" \
-          --set profile.s3.secretKey="wJalrXUtnFEMI!K7MDENG!bPxRfiCYEXAMPLEKEY" \
+          --set profile.location.type='s3Compliant' \
+          --set profile.location.bucket='kanister-bucket' \
+          --set profile.location.endpoint='https://my-custom-s3-provider:9000' \
+          --set profile.aws.accessKey='AKIAIOSFODNN7EXAMPLE' \
+          --set profile.aws.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
           --set kanister.controller_namespace="kanister"
 
 

docs/install.rst

@@ -35,7 +35,7 @@ This will install the Kanister controller in the `kanister` namespace
    $ helm repo add kanister https://charts.kanister.io/
 
    # Install the Kanister operator controller using helm
-   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.15.0
+   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.16.0
 
    # Create an S3 Compliant Kanister profile using kanctl
    $ kanctl create profile s3compliant --bucket <bucket> --access-key ${AWS_ACCESS_KEY_ID} \
@@ -73,5 +73,5 @@ namespace.
 
 .. code-block:: bash
 
-   # Deploy controller version 0.15.0 to Kubernetes
-   $ make deploy VERSION="0.15.0"
+   # Deploy controller version 0.16.0 to Kubernetes
+   $ make deploy VERSION="0.16.0"

docs/overview.rst

@@ -64,7 +64,7 @@ and backup to an AWS S3 bucket.
   helm repo add kanister http://charts.kanister.io
 
   # Install the Kanister Controller
-  helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.15.0
+  helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.16.0
 
   # Install MySQL and configure its Kanister Blueprint.
   # Also create a Profile CR that can be used in ActionSets

docs/tooling.rst

@@ -375,7 +375,7 @@ The following snippet is an example of using kando from inside a Blueprint.
 
   kando location delete --profile '{{ .Profile }}' --path '/backup/path'
 
-  kando output version 0.15.0
+  kando output version 0.16.0
 
 Install the tools
 =================
@@ -398,7 +398,7 @@ These tools, especially `kando` are meant to be invoked inside containers via
 Blueprints. Although suggest using the released image when possible, we've also
 made it simple to add these tools to your container.
 
-The released image, `kanisterio/kanister-tools:0.15.0`, is hosted by
+The released image, `kanisterio/kanister-tools:0.16.0`, is hosted by
 `dockerhub <https://cloud.docker.com/swarm/kanisterio/repository/docker/kanisterio/kanister-tools/general>`_.
 
 The Dockerfile for this image is in the

examples/helm/kanister/kanister-elasticsearch/Chart.yaml

@@ -1,6 +1,6 @@
 name: kanister-elasticsearch
 home: https://www.elastic.co/products/elasticsearch
-version: 1.3.0
+version: 0.16.0
 appVersion: 6.3.1
 description: Elasticsearch w/ Kanister support based on incubator/elasticsearch
 icon: https://static-www.elastic.co/assets/blteb1c97719574938d/logo-elastic-elasticsearch-lt.svg
@@ -13,9 +13,9 @@ sources:
 - https://github.com/clockworksoul/helm-elasticsearch
 - https://github.com/pires/kubernetes-elasticsearch-cluster
 maintainers:
-- name: Tom Manville
+- name: tdmanv
   email: tom@kasten.io
-- name: Ilya Kislenko
+- name: depohmel
   email: ilya@kasten.io
-- name: Supriya Kharade
+- name: SupriyaKasten
   email: supriya@kasten.io

examples/helm/kanister/kanister-elasticsearch/README.md

@@ -13,7 +13,7 @@ If you want to avoid doing that upgrade to Elasticsearch 5.6 first before moving
 
 * Kubernetes 1.9+ with Beta APIs enabled.
 * PV support on the underlying infrastructure.
-* Kanister version 0.7.0 with `profiles.cr.kanister.io` CRD installed
+* Kanister version 0.16.0 with `profiles.cr.kanister.io` CRD installed
 
 ## StatefulSets Details
 * https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/
@@ -39,21 +39,24 @@ $ helm repo add kanister http://charts.kanister.io
 ```
 
 Then install the sample Elasticsearch application with the release name `my-release` in its own namespace
-`es-test` using the following command:
+`es-test` using the command below. Make sure you have the kanister controller running in namespace `kasten-io` which is the default setting in Elasticsearch charts. Otherwise, you will also have to set the `kanister.controller_namespace` parameter value to the respective kanister controller namespace in the following command:
 
 ```bash
+# Replace the default s3 credentials (endpoint, bucket and region) with your credentials before you run this command
 $ helm install kanister/kanister-elasticsearch -n my-release --namespace es-test \
      --set profile.create='true' \
      --set profile.profileName='es-test-profile' \
-     --set profile.s3.endpoint='https://my-custom-s3-provider:9000' \
-     --set profile.s3.accessKey='AKIAIOSFODNN7EXAMPLE' \
-     --set profile.s3.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
-     --set profile.s3.bucket='kanister-bucket'
+     --set profile.location.type='s3Compliant' \
+     --set profile.location.bucket='kanister-bucket' \
+     --set profile.location.endpoint='https://my-custom-s3-provider:9000' \
+     --set profile.location.region=us-west-2 \
+     --set profile.aws.accessKey="${AWS_ACCESS_KEY_ID}" \
+     --set profile.aws.secretKey="${AWS_SECRET_ACCESS_KEY}"
 ```
 
 The command deploys Elasticsearch on the Kubernetes cluster in the default
 configuration. The [configuration](#configuration) section lists the parameters that can be
-configured during installation.
+configured during installation. It also installs a `profiles.cr.kanister.io` CRD named `es-test-profile` in `es-test` namespace.
 
 The command will also configure a location where artifacts resulting from Kanister
 data operations such as backup should go. This is stored as a `profiles.cr.kanister.io`
@@ -63,8 +66,6 @@ not. Support for creating an ActionSet as part of install is simply for convenie
 This CR can be shared between Kanister-enabled application instances so one option is to
 only create as part of the first instance.
 
-Make sure you have the kanister controller running in namespace `kasten-io` which is the default setting in Elasticsearch otherwise, you will also have to set the `kanister.controller_namespace` parameter value to the respective kanister controller namespace in the above command.
-
 If not creating a Profile CR, it is possible to use an even simpler command.
 
 ```bash
@@ -92,33 +93,17 @@ green  open   customer xbwj34pTSZOdDI7xVR0qIA   5   1          1            0
 
 ## Protect the Application
 
-You can now take a backup of the Elasticsearch data using an ActionSet defining backup for this application. Create an ActionSet in the same namespace as the controller.
+You can now take a backup of the Elasticsearch data using an ActionSet defining backup for this application. Create an ActionSet in the same namespace as the controller using `kanctl`, a command-line tool that helps create ActionSets as shown below:
 
 ```bash
-$ cat << EOF | kubectl create -f -
-apiVersion: cr.kanister.io/v1alpha1
-kind: ActionSet
-metadata:
-  name: es-backup-1
-  namespace: kasten-io
-spec:
-  actions:
-  - name: backup
-    blueprint: my-release-kanister-elasticsearch-blueprint
-    object:
-      kind: StatefulSet
-      name: my-release-kanister-elasticsearch-data
-      namespace: es-test
-    profile:
-      apiVersion: v1alpha1
-      kind: Profile
-      name: default-profile
-      namespace: kasten-io
-EOF
+$ kanctl create actionset --action backup --namespace kasten-io --blueprint my-release-kanister-elasticsearch-blueprint --statefulset es-test/my-release-kanister-elasticsearch-data --profile es-test/es-test-profile
 
-$ kubectl --namespace kanister get actionsets.cr.kanister.io
+$ kubectl --namespace kasten-io get actionsets.cr.kanister.io
 NAME                AGE
-es-backup-1         2h
+backup-lphk7        2h
+
+# View the status of the actionset
+$ kubectl --namespace kasten-io describe actionset backup-lphk7
 ```
 
 ## Disaster strikes!
@@ -144,11 +129,11 @@ health status index uuid pri rep docs.count docs.deleted store.size pri.store.si
 To restore the missing data, we want to use the backup created earlier in the steps above. An easy way to do this is to leverage `kanctl`, a command-line tool that helps create ActionSets that depend on other ActionSets:
 
 ```bash
-$ kanctl --namespace kasten-io create actionset --action restore --from "es-backup-1"
-actionset restore-es-backup-1-mqsm5 created
+$ kanctl --namespace kasten-io create actionset --action restore --from "backup-lphk7"
+actionset restore-backup-lphk7-hndm6 created
 
 # View the status of the ActionSet
-kubectl --namespace kasten-io get actionset restore-es-backup-1-mqsm5 -oyaml
+kubectl --namespace kasten-io describe actionset restore-backup-lphk7-hndm6
 ```
 
 You should now see that the data has been successfully restored to Elasticsearch!
@@ -164,11 +149,19 @@ green  open   customer xbwj34pTSZOdDI7xVR0qIA   5   1          1            0
 The artifacts created by the backup action can be cleaned up using the following command:
 
 ```bash
-$ kanctl --namespace kasten-io create actionset --action delete --from "es-backup-1"
-actionset "delete-es-backup-1-fnb9v" created
+$ kanctl --namespace kasten-io create actionset --action delete --from "backup-lphk7"
+actionset "delete-backup-lphk7-5n8nz" created
 
 # View the status of the ActionSet
-$ kubectl --namespace kasten-io get actionset delete-es-backup-1-fnb9v -oyaml
+$ kubectl --namespace kasten-io describe actionset delete-backup-lphk7-5n8nz
+```
+
+### Troubleshooting
+
+If you run into any issues with the above commands, you can check the logs of the controller using:
+
+```bash
+$ kubectl --namespace kasten-io logs -l app=kanister-operator
 ```
 
 ## Delete the Helm deployment as normal
@@ -193,11 +186,11 @@ default values. The Profile CR parameters are passed to the profile sub-chart.
 | `profile.create` | (Optional) Specify if a Profile CR should be created as part of install. | ``false`` |
 | `profile.defaultProfile` | (Optional if not creating a default Profile) Set to ``true`` to create a profile with name `default-profile` | ``false`` |
 | `profile.profileName` | (Required if not creating a default Profile) Name for the profile that is created | `nil` |
-| `profile.s3.accessKey` | (Required if creating profile) API Key for an s3 compatible object store. | `nil`|
-| `profile.s3.secretKey` | (Required if creating profile) Corresponding secret for `accessKey`. | `nil` |
-| `profile.s3.bucket` | (Required if creating profile) A bucket that will be used to store Kanister artifacts. <br><br>The bucket must already exist and the account with the above API key and secret needs to have sufficient permissions to list, get, put, delete. | `nil` |
-| `profile.s3.region` | (Optional if creating profile) Region to be used for the bucket. | `nil` |
-| `profile.s3.endpoint` | (Optional if creating profile) The URL for an s3 compatible object store provider. Can be omitted if provider is AWS. Required for any other provider. | `nil` |
+| `profile.aws.accessKey` | (Required if creating profile) API Key for an s3 compatible object store. | `nil`|
+| `profile.aws.secretKey` | (Required if creating profile) Corresponding secret for `accessKey`. | `nil` |
+| `profile.location.bucket` | (Required if creating profile) A bucket that will be used to store Kanister artifacts. <br><br>The bucket must already exist and the account with the above API key and secret needs to have sufficient permissions to list, get, put, delete. | `nil` |
+| `profile.location.region` | (Optional if creating profile) Region to be used for the bucket. | `nil` |
+| `profile.location.endpoint` | (Optional if creating profile) The URL for an s3 compatible object store provider. Can be omitted if provider is AWS. Required for any other provider. | `nil` |
 | `profile.verifySSL` | (Optional if creating profile) Set to ``false`` to disable SSL verification on the s3 endpoint. | `true` |
 | `kanister.controller_namespace` | (Optional) Specify the namespace where the Kanister controller is running. | kasten-io |
 
@@ -332,8 +325,10 @@ $ helm install kanister/kanister-elasticsearch -n my-release --namespace es-test
      --set data.storage=100Gi \
      --set profile.create='true' \
      --set profile.profileName='es-test-profile' \
-     --set profile.s3.endpoint='https://my-custom-s3-provider:9000' \
-     --set profile.s3.accessKey='AKIAIOSFODNN7EXAMPLE' \
-     --set profile.s3.secretKey='wJalrXUtnFEMI%K7MDENG%bPxRfiCYEXAMPLEKEY' \
-     --set profile.s3.bucket='kanister-bucket'
+     --set profile.location.type='s3Compliant' \
+     --set profile.location.bucket='kanister-bucket' \
+     --set profile.location.endpoint='https://my-custom-s3-provider:9000' \
+     --set profile.location.region=us-west-2 \
+     --set profile.aws.accessKey="${AWS_ACCESS_KEY_ID}" \
+     --set profile.aws.secretKey="${AWS_SECRET_ACCESS_KEY}"
 ```

examples/helm/kanister/kanister-elasticsearch/ci/lint-values.yaml

@@ -0,0 +1,7 @@
+profile:
+  create: true
+  defaultProfile: true
+  s3:
+    accessKey: SomeBogusTestKey=
+    secretKey: SomeTestSecrest==
+    bucket: linttestname

examples/helm/kanister/kanister-elasticsearch/image/Dockerfile

@@ -1,5 +1,5 @@
 
-FROM kanisterio/kanister-tools:0.15.0
+FROM kanisterio/kanister-tools:0.16.0
 MAINTAINER Supriya Kharade <supriya@kasten.io>
 
 COPY esdump-setup.sh /esdump-setup.sh

examples/helm/kanister/kanister-elasticsearch/kanister/elasticsearch-blueprint.yaml

@@ -10,7 +10,7 @@ actions:
       name: takeDataSnapshot
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "kanisterio/es-sidecar:0.15.0"
+        image: "kanisterio/es-sidecar:0.16.0"
         command:
           - bash
           - -o
@@ -37,7 +37,7 @@ actions:
       name: restoreSnapshots
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "kanisterio/es-sidecar:0.15.0"
+        image: "kanisterio/es-sidecar:0.16.0"
         command:
           - bash
           - -o
@@ -63,7 +63,7 @@ actions:
       name: deleteSnapshots
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "kanisterio/es-sidecar:0.15.0"
+        image: "kanisterio/es-sidecar:0.16.0"
         command:
           - bash
           - -o

examples/helm/kanister/kanister-elasticsearch/requirements.yaml

@@ -1,5 +1,5 @@
 dependencies:
   - name: profile
-    version: "0.15.0"
+    version: "0.16.0"
     condition: profile.create
     repository: "https://charts.kanister.io"