diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..9b0d1685ee --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +SHELL := /bin/bash -euo pipefail + +config_file_name = Kube-manifest.yaml +example_config = docs/$(config_file_name) + +.PHONY: all +all: docs + +# In a branch, run 'make docs' to update docs with +# generated code, then merge it to master. +docs: $(example_config) + +# Use kustomize to create the standard kustomize configuration +# file that appears in the website's documentation. +$(example_config): /tmp/bin/kustomize + rm -f TMP + echo "# This is a generated example; do not edit. Rebuild with 'make docs'." >> TMP + echo " " >> TMP + /tmp/bin/kustomize init + cat $(config_file_name) >> TMP + mv TMP $(example_config) + rm $(config_file_name) + +/tmp/bin/kustomize: + go build -o /tmp/bin/kustomize kustomize.go diff --git a/README.md b/README.md new file mode 100644 index 0000000000..c3c474f3e6 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +# kustomize + +[applied]: docs/glossary.md#apply +[base]: docs/glossary.md#base +[declarative configuration]: docs/glossary.md#declarative-application-management +[demo]: demos/README.md +[imageBase]: docs/base.jpg +[imageOverlay]: docs/overlay.jpg +[manifest]: docs/glossary.md#manifest +[overlay]: docs/glossary.md#overlay +[resources]: docs/glossary.md#resource +[workflows]: docs/workflows.md + +`kustomize` is a command line tool supporting +template-free customization of declarative +configuration targetted to kubernetes. + +## Installation + +Assumes [Go](https://golang.org/) is installed +and your `PATH` contains `$GOPATH/bin`: + + +``` +go get k8s.io/kubectl/cmd/kustomize +``` + +## Usage + +#### 1) Make a base + +A [base] configuration is a [manifest] listing a set of +k8s [resources] - deployments, services, configmaps, +secrets that serve some common purpose. + +![base image][imageBase] + +#### 2) Customize it with overlays + +An [overlay] customizes your base along different dimensions +for different purposes or different teams, e.g. for +_development, staging and production_. + +![overlay image][imageOverlay] + +#### 3) Run kustomize + +Run kustomize on your overlay. The result +is printed to `stdout` as a set of complete +resources, ready to be [applied] to a cluster. + +For more details, try a [demo]. diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000000..95ba60c790 --- /dev/null +++ b/build/README.md @@ -0,0 +1,16 @@ +## Steps to run build locally + +Install container-builder-local from github and define GOOS, GOARCH, OUTPUT env +variables and run following command + +```sh +container-builder-local --config=cmd/kustomize/build/cloudbuild_local.yaml --dryrun=false --substitutions=_GOOS=$GOOS,_GOARCH=$GOARCH --write-workspace=$OUTPUT . +``` + +## Steps to submit build to Google container builder + +You need to be at the repo level to be able to run the following command + +``` +gcloud container builds submit . --config=cmd/kustomize/build/cloudbuild.yaml --substitutions=_GOOS=$GOOS,_GOARCH=$GOARCH +``` diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 0000000000..a557fcf2a8 --- /dev/null +++ b/build/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +# Google Container Builder automatically checks out all the code under the /workspace directory, +# but we actually want it to under the correct expected package in the GOPATH (/go) +# - Create the directory to host the code that matches the expected GOPATH package locations +# - Use /go as the default GOPATH because this is what the image uses +# - Link our current directory (containing the source code) to the package location in the GOPATH +export PKG=k8s.io +export REPO=kubectl +export CMD=kustomize + +mkdir -p /go/src/$PKG +ln -s $(pwd) /go/src/$PKG/$REPO + +# Create the output directory for the binaries we will build +# Make sure CGO is 0 so the binaries are statically compiled and linux which is necessary for cross compiling go +export CGO=0 +export DEST=/workspace/_output/$CMD/bin +mkdir -p $DEST || echo "" + +go build -o $DEST/$CMD $PKG/$REPO/cmd/$CMD + +# Explicitly set the values of the variables in package "X" using ldflag so that they are statically compiled into +# the "version" command +export GITCOMMIT=$(git rev-parse HEAD) +export BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') +export X=$PKG/$REPO/cmd/$CMD/version +go build -o $DEST/$CMD \ + -ldflags "-X $X.kustomizeVersion=$TAG -X $X.goos=$GOOS -X $X.goarch=$GOARCH -X $X.gitCommit=$GITCOMMIT -X $X.buildDate=$BUILDDATE" \ + $PKG/$REPO/cmd/$CMD + +# Generate the tar archive +cd /workspace/_output/ +tar -czvf /workspace/$CMD-$VERSION-$GOOS-$GOARCH.tar.gz $CMD diff --git a/build/cloudbuild.yaml b/build/cloudbuild.yaml new file mode 100644 index 0000000000..a0b63264a6 --- /dev/null +++ b/build/cloudbuild.yaml @@ -0,0 +1,30 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO(droot): add instructions for running in production. +steps: +- name: "ubuntu" + args: ["mkdir", "-p", "/workspace/_output"] +- name: "golang:1.10-stretch" + args: ["bash", "cmd/kustomize/build/build.sh"] + env: + - 'GOOS=${_GOOS}' + - 'GOARCH=${_GOARCH}' + - 'VERSION=${TAG_NAME}' +- name: 'gcr.io/cloud-builders/gsutil' + args: ['-h', 'Content-Type:application/gzip', 'cp', '-a', 'public-read', 'kustomize-${TAG_NAME}-${_GOOS}-${_GOARCH}.tar.gz', 'gs://kustomize-release/kustomize-${TAG_NAME}-${_GOOS}-${_GOARCH}.tar.gz'] + env: + - 'GOOS=${_GOOS}' + - 'GOARCH=${_GOARCH}' + - 'VERSION=${TAG_NAME}' diff --git a/build/cloudbuild_local.yaml b/build/cloudbuild_local.yaml new file mode 100644 index 0000000000..66f9b87e52 --- /dev/null +++ b/build/cloudbuild_local.yaml @@ -0,0 +1,31 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Instructions to run locally: +# Download google container builder: https://github.com/kubernetes-sigs/container-builder-local +# Set you GOOS and GOARCH vars to match your system +# Set OUTPUT to the location to write the directory containing the tar.gz +# $ container-builder-local --config=build/cloudbuild_local.yaml --dryrun=false \ +# --substitutions=_GOOS=$GOOS,_GOARCH=$GOARCH --write-workspace=$OUTPUT . +# Release tar will be in $OUTPUT + +steps: +- name: "ubuntu" + args: ["mkdir", "-p", "/workspace/_output"] +- name: "golang:1.10-stretch" + args: ["bash", "cmd/kustomize/build/build.sh"] + env: + - 'GOOS=${_GOOS}' + - 'GOARCH=${_GOARCH}' + - 'VERSION=${TAG_NAME}' diff --git a/demos/README.md b/demos/README.md new file mode 100644 index 0000000000..e3818f1b94 --- /dev/null +++ b/demos/README.md @@ -0,0 +1,13 @@ +# Demos + +These demos are covered by presubmit tests. + + * [hello world](helloWorld.md) - Deploy multiple + (differently configured) instances of a simple Hello + World server. + + * [mysql](mySql.md) - Create a mySql production + configuration from scratch. + + * [springboot](springboot.md) - Create a Spring Boot application production + configuration from scratch. diff --git a/demos/helloWorld.md b/demos/helloWorld.md new file mode 100644 index 0000000000..d6e9c5bf5c --- /dev/null +++ b/demos/helloWorld.md @@ -0,0 +1,432 @@ +[base]: ../docs/glossary.md#base +[config]: https://github.com/kinflate/example-hello +[hello]: https://github.com/monopole/hello +[instance]: ../docs/glossary.md#instance +[instances]: ../docs/glossary.md#instance +[manifest]: ../docs/glossary.md#manifest +[original]: https://github.com/kinflate/example-hello +[overlay]: ../docs/glossary.md#overlay +[overlays]: ../docs/glossary.md#overlay + +# Demo: hello world with instances + +Steps: + + 1. Clone an existing configuration as a [base]. + 1. Customize it. + 1. Create two different [overlays] (_staging_ and _production_) + from the customized base. + 1. Run kustomize and kubectl to deploy staging and production. + +First define a place to work: + + +``` +DEMO_HOME=$(mktemp -d) +``` + +Alternatively, use + +> ``` +> DEMO_HOME=~/hello +> ``` + +## Clone an example + +Let's run the [hello] service. +Here's an existing [config] for it. + +Clone this config to a directory called `base`: + + +``` +git clone \ + https://github.com/kinflate/example-hello \ + $DEMO_HOME/base +``` + + +``` +tree $DEMO_HOME +``` + +Expecting something like: + +> ``` +> /tmp/tmp.IyYQQlHaJP +> └── base +> ├── configMap.yaml +> ├── deployment.yaml +> ├── kustomize.yaml +> ├── LICENSE +> ├── README.md +> └── service.yaml +> ``` + + +One could immediately apply these resources to a +cluster: + +> ``` +> kubectl apply -f $DEMO_HOME/base +> ``` + +to instantiate the _hello_ service. `kubectl` +would only recognize the resource files. + +## The Base Manifest + +The `base` directory has a [manifest] file: + + +``` +BASE=$DEMO_HOME/base +more $BASE/kustomize.yaml +``` + +Run `kustomize` on the base to emit customized resources +to `stdout`: + + +``` +kustomize build $BASE +``` + +## Customize the base + +A first customization step could be to change the _app +label_ applied to all resources: + + +``` +sed -i 's/app: hello/app: my-hello/' \ + $BASE/kustomize.yaml +``` + +See the effect: + +``` +kustomize build $BASE | grep -C 3 app: +``` + +## Create Overlays + +Create a _staging_ and _production_ [overlay]: + + * _Staging_ enables a risky feature not enabled in production. + * _Production_ has a higher replica count. + * Web server greetings from these cluster + [instances] will differ from each other. + + +``` +OVERLAYS=$DEMO_HOME/overlays +mkdir -p $OVERLAYS/staging +mkdir -p $OVERLAYS/production +``` + +#### Staging Manifest + +In the `staging` directory, make a manifest +defining a new name prefix, and some different labels. + + +``` +cat <<'EOF' >$OVERLAYS/staging/kustomize.yaml +apiVersion: manifest.k8s.io/v1alpha1 +kind: Package +metadata: + name: makes-staging-hello +namePrefix: staging- +objectLabels: + instance: staging + org: acmeCorporation +objectAnnotations: + note: Hello, I am staging! +bases: +- ../../base +patches: +- map.yaml +EOF +``` + +#### Staging Patch + +Add a configMap customization to change the server +greeting from _Good Morning!_ to _Have a pineapple!_ + +Also, enable the _risky_ flag. + + +``` +cat <$OVERLAYS/staging/map.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: the-map +data: + altGreeting: "Have a pineapple!" + enableRisky: "true" +EOF +``` + +#### Production Manifest + +In the production directory, make a manifest +with a different name prefix and labels. + + +``` +cat <$OVERLAYS/production/kustomize.yaml +apiVersion: manifest.k8s.io/v1alpha1 +kind: Package +metadata: + name: makes-production-tuthello +namePrefix: production- +objectLabels: + instance: production + org: acmeCorporation +objectAnnotations: + note: Hello, I am production! +bases: +- ../../base +patches: +- deployment.yaml +EOF +``` + + +#### Production Patch + +Make a production patch that increases the replica +count (because production takes more traffic). + + +``` +cat <$OVERLAYS/production/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: the-deployment +spec: + replicas: 10 +EOF +``` + +## Compare overlays + + +`DEMO_HOME` now contains: + + - a _base_ directory - a slightly customized clone + of the original configuration, and + + - an _overlays_ directory, containing the manifests + and patches required to create distinct _staging_ + and _production_ instances in a cluster. + +Review the directory structure and differences: + + +``` +tree $DEMO_HOME +``` + +Expecting something like: + +> ``` +> /tmp/tmp.IyYQQlHaJP1 +> ├── base +> │   ├── configMap.yaml +> │   ├── deployment.yaml +> │   ├── kustomize.yaml +> │   ├── LICENSE +> │   ├── README.md +> │   └── service.yaml +> └── overlays +> ├── production +> │   ├── deployment.yaml +> │   └── kustomize.yaml +> └── staging +> ├── kustomize.yaml +> └── map.yaml +> ``` + +Compare the output directly +to see how _staging_ and _production_ differ: + + +``` +diff \ + <(kustomize build $OVERLAYS/staging) \ + <(kustomize build $OVERLAYS/production) |\ + more +``` + +The first part of the difference output should look +something like + +> ```diff +> < altGreeting: Have a pineapple! +> < enableRisky: "true" +> --- +> > altGreeting: Good Morning! +> > enableRisky: "false" +> 8c8 +> < note: Hello, I am staging! +> --- +> > note: Hello, I am production! +> 11c11 +> < instance: staging +> --- +> > instance: production +> 13c13 +> (...truncated) +> ``` + + +## Deploy + +The individual resource sets are: + + +``` +kustomize build $OVERLAYS/staging +``` + + +``` +kustomize build $OVERLAYS/production +``` + +To deploy, pipe the above commands to kubectl apply: + +> ``` +> kustomize build $OVERLAYS/staging |\ +> kubectl apply -f - +> ``` + +> ``` +> kustomize build $OVERLAYS/production |\ +> kubectl apply -f - +> ``` + +## Rolling updates + +### Review + +The _hello-world_ deployment running in this cluster is +configured with data from a configMap. + +The deployment refers to this map by name: + + + +``` +grep -C 2 configMapKeyRef $DEMO_HOME/base/deployment.yaml +``` + +Changing the data held by a live configMap in a cluster +is considered bad practice. Deployments have no means +to know that the configMaps they refer to have +changed, so such updates have no effect. + +The recommended way to change a deployment's +configuration is to + + 1. create a new configMap with a new name, + 1. patch the _deployment_, modifying the name value of + the appropriate `configMapKeyRef` field. + +This latter change initiates rolling update to the pods +in the deployment. The older configMap, when no longer +referenced by any other resource, is eventually garbage +collected. + +### How this works with kustomize + +[patch]: ../docs/glossary.md#patch + +The _staging_ instance here has a configMap [patch]: + + +``` +cat $OVERLAYS/staging/map.yaml +``` + +This patch is by definition a named but not necessarily +complete resource spec intended to modify a complete +resource spec. + +The resource it modifies is here: + + +``` +cat $DEMO_HOME/base/configMap.yaml +``` + +For a patch to work, the names in the `metadata/name` +fields must match. + +However, the name values specified in the file are +_not_ what gets used in the cluster. By design, +kustomize modifies these names. To see the names +ultimately used in the cluster, just run kustomize: + + +``` +kustomize build $OVERLAYS/staging |\ + grep -B 8 -A 1 staging-the-map +``` + +The configMap name is prefixed by _staging-_, per the +`namePrefix` field in +`$OVERLAYS/staging/kustomize.yaml`. + +The suffix to the configMap name is generated from a +hash of the maps content - in this case the name suffix +is _hhhhkfmgmk_: + + +``` +kustomize build $OVERLAYS/staging | grep hhhhkfmgmk +``` + +Now modify the map patch, to change the greeting +the server will use: + + +``` +sed -i 's/pineapple/kiwi/' $OVERLAYS/staging/map.yaml +``` + +Run kustomize again to see the new names: + + +``` +kustomize build $OVERLAYS/staging |\ + grep -B 8 -A 1 staging-the-map +``` + +Confirm that the change in configMap content resulted +in three new names ending in _khk45ktkd9_ - one in the +configMap name itself, and two in the deployment that +uses the map: + + +``` +test 3 == $(kustomize build $OVERLAYS/staging | grep khk45ktkd9 | wc -l) +``` + +Applying these resources to the cluster will result in +a rolling update of the deployments pods, retargetting +them from the _hhhhkfmgmk_ maps to the _khk45ktkd9_ +maps. The system will later garbage collect the +unused maps. + +## Rollback + +To rollback, one would undo whatever edits were made to +the configuation in source control, then rerun kustomize +on the reverted configuration and apply it to the +cluster. diff --git a/demos/mySql.md b/demos/mySql.md new file mode 100644 index 0000000000..e7faee9d20 --- /dev/null +++ b/demos/mySql.md @@ -0,0 +1,239 @@ +# Demo: MySql + +This example takes some off-the-shelf k8s resources +designed for MySQL, and customizes them to suit a +production scenario. + +In the production environment we want: + +- MySQL resource names to be prefixed by 'prod-'. +- MySQL resources to have 'env: prod' labels. +- MySQL to use persistent disk for storing data. + +### Download resources + +Download `deployment.yaml`, `service.yaml` and +`secret.yaml`. These are plain k8s resources files one +could add to a k8s cluster to run MySql. + + +``` +DEMO_HOME=$(mktemp -d) +cd $DEMO_HOME + +# Get MySQL configs +for f in service secret deployment ; do \ + wget https://raw.githubusercontent.com/kinflate/mysql/master/emptyDir/$f.yaml ; \ +done +``` + +### Initialize a manifest + +A _manifest_ groups these resources together. + +Create one: + + +``` +cd $DEMO_HOME +kustomize init +``` + +You should now have a file called `kustomize.yaml`: + + +``` +cat $DEMO_HOME/kustomize.yaml +``` + +containing something like: + + +> ``` +> apiVersion: manifest.k8s.io/v1alpha1 +> kind: Manifest +> metadata: +> name: helloworld +> # description: helloworld does useful stuff. +> namePrefix: some-prefix +> # Labels to add to all objects and selectors. +> # These labels would also be used to form the selector for apply --prune +> # Named differently than “labels” to avoid confusion with metadata for this object +> objectLabels: +> app: helloworld +> objectAnnotations: +> note: This is a example annotation +> resources: +> - deployment.yaml +> - service.yaml +> # There could also be configmaps in Base, which would make these overlays +> configmaps: [] +> # There could be secrets in Base, if just using a fork/rebase workflow +> secrets: [] +> recursive: true +> ``` + + +### Add the resources to the manifest + + +``` +cd $DEMO_HOME + +kustomize edit add resource secret.yaml +kustomize edit add resource service.yaml +kustomize edit add resource deployment.yaml + +cat kustomize.yaml +``` + +`kustomize.yaml`'s resources section should contain: + +> ``` +> apiVersion: manifest.k8s.io/v1alpha1 +> .... +> resources: +> - secret.yaml +> - service.yaml +> - deployment.yaml +> ``` + +### Name Customization + +Arrange for the MySQL resources to begin with prefix +_prod-_ (since they are meant for the _production_ +environment): + + +``` +cd $DEMO_HOME + +kustomize edit set nameprefix 'prod-' + +cat kustomize.yaml +``` + +`kustomize.yaml` should have updated value of namePrefix field: + +> ``` +> apiVersion: manifest.k8s.io/v1alpha1 +> .... +> namePrefix: prod- +> objectAnnotations: +> note: This is a example annotation +> ``` + +This `namePrefix` directive adds _prod-_ to all +resource names. + + +``` +kustomize build $DEMO_HOME +``` + +The output should contain: +> ``` +> apiVersion: v1 +> data: +> password: YWRtaW4= +> kind: Secret +> metadata: +> .... +> name: prod-mysql-pass-d2gtcm2t2k +> --- +> apiVersion: v1 +> kind: Service +> metadata: +> .... +> name: prod-mysql +> spec: +> .... +> --- +> apiVersion: apps/v1beta2 +> kind: Deployment +> metadata: +> .... +> name: prod-mysql +> spec: +> selector: +> .... +> ``` + +### Label Customization + +We want resources in production environment to have +certain labels so that we can query them by label +selector. + +`kustomize` does not have `set label` command to add +label, but we can edit `kustomize.yaml` file under +`prod` directory and add the production labels under +`objectLabels` fields as highlighted below. + + +``` +sed -i 's/app: helloworld/app: prod/' \ + $DEMO_HOME/kustomize.yaml +``` + +At this point, running `kustomize build` will +generate MySQL configs with name-prefix 'prod-' and +labels `env:prod`. + +### Storage customization + +Off the shelf MySQL uses `emptyDir` type volume, which +gets wiped away if the MySQL Pod is recreated, and that +is certainly not desirable for production +environment. So we want to use Persistent Disk in +production. kustomize lets you apply `patches` to the +resources. + + +``` +cat <<'EOF' > $DEMO_HOME/persistent-disk.yaml +apiVersion: apps/v1beta2 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: mysql +spec: + template: + spec: + volumes: + - name: mysql-persistent-storage + emptyDir: null + gcePersistentDisk: + pdName: mysql-persistent-storage +EOF +``` + +Specify the patch file in the manifest: + + +``` +cat <<'EOF' >> $DEMO_HOME/kustomize.yaml +patches: +- persistent-disk.yaml +EOF +``` + +Lets break this down: + +- In the first step, we created a YAML file named + `persistent-disk.yaml` to patch the resource defined + in deployment.yaml + +- Then we added `persistent-disk.yaml` to list of + `patches` in `kustomize.yaml`. `kustomize build` + will apply this patch to the deployment resource with + the name `mysql` as defined in the patch. + + +The output of the following command can now be applied +to the cluster (i.e. piped to `kubectl apply`) to +create the production environment. + + +``` +kustomize build $DEMO_HOME # | kubectl apply -f - +``` diff --git a/demos/springboot.md b/demos/springboot.md new file mode 100644 index 0000000000..a71e3e584b --- /dev/null +++ b/demos/springboot.md @@ -0,0 +1,350 @@ +# Demo: SpringBoot + +In this tutorial, you will learn - how to use `kustomize` to customize a basic Spring Boot application's +k8s configuration for production use cases. + +In the production environment we want to customize the following: + +- add application specific configuration for this Spring Boot application +- configure prod DB access configuration +- resource names to be prefixed by 'prod-'. +- resources to have 'env: prod' labels. +- JVM memory to be properly set. +- health check and readiness check. + +### Download resources + +Download `deployment.yaml`, `service.yaml`. These are plain k8s resources files one +could add to a k8s cluster to run sbdemo. + + +``` +DEMO_HOME=$(mktemp -d) +cd $DEMO_HOME + +# Get SpringBoot configs +for f in service deployment; do \ + wget https://raw.githubusercontent.com/kinflate/example-springboot/master/$f.yaml ; \ +done +``` + +### Initialize a manifest + +A _manifest_ groups these resources together. + +Create one: + + +``` +cd $DEMO_HOME +kustomize init +``` + +The above step will create a `kustomize` configuration file called `kustomize.yaml` in current directory. + + +``` +cat $DEMO_HOME/kustomize.yaml +``` + +containing something like: + + +> ``` +> apiVersion: manifest.k8s.io/v1alpha1 +> kind: Manifest +> metadata: +> name: helloworld +> # description: helloworld does useful stuff. +> namePrefix: some-prefix +> # Labels to add to all objects and selectors. +> # These labels would also be used to form the selector for apply --prune +> # Named differently than “labels” to avoid confusion with metadata for this object +> objectLabels: +> app: helloworld +> objectAnnotations: +> note: This is a example annotation +> resources: +> - deployment.yaml +> - service.yaml +> # There could also be configmaps in Base, which would make these overlays +> configMapGenerator: [] +> # There could be secrets in Base, if just using a fork/rebase workflow +> secretGenerator: [] +> ``` + + +### Add the resources to the manifest + + +``` +cd $DEMO_HOME + +kustomize edit add resource service.yaml +kustomize edit add resource deployment.yaml + +cat kustomize.yaml +``` + +`kustomize.yaml`'s resources section should contain: + +> ``` +> apiVersion: manifest.k8s.io/v1alpha1 +> .... +> resources: +> - service.yaml +> - deployment.yaml +> ``` + +### Add configmap to the manifest + +``` +cd $DEMO_HOME +wget https://raw.githubusercontent.com/kinflate/example-springboot/master/application.properties +kustomize edit add configmap demo-configmap --from-file application.properties + +cat kustomize.yaml +``` +`kustomize.yaml`'s configMapGenerator section should contain: +> ``` +> configMapGenerator: +> - files: +> - application.properties +> name: demo-configmap +> kind: Manifest +> ``` + +### Customize configmap +We want to add database credentials for the prod environment. In general, these credentials can be put into the file `application.properties`. +However, for some cases, we want to keep the credentials in a different file and keep application specific configs in `application.properties`. + With this clear separation, the credentials and application specific things can be managed and maintained flexibly by different teams. +For example, application developers only tune the application configs in `application.properties` and operation teams or SREs +only care about the credentials. + +For Spring Boot application, we can set an active profile through the environment variable `spring.profiles.active`. Then +the application will pick up an extra `application-.properties` file. With this, we can customize the configmap in two +steps. Add an environment variable through the patch and add a file to the configmap. + +``` +cat <$DEMO_HOME/patch.yaml +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: sbdemo +spec: + template: + spec: + containers: + - name: sbdemo + env: + - name: spring.profiles.active + value: prod +EOF + +cat <>$DEMO_HOME/kustomize.yaml +patches: +- patch.yaml +EOF + +cat <$DEMO_HOME/application-prod.properties +spring.jpa.hibernate.ddl-auto=update +spring.datasource.url=jdbc:mysql://:3306/db_example +spring.datasource.username=root +spring.datasource.password=admin +EOF + +kustomize edit add configmap demo-configmap --from-file application-prod.properties + +cat kustomize.yaml +``` +`kustomize.yaml`'s configMapGenerator section should contain: +> ``` +> configMapGenerator: +> - files: +> - application.properties +> - application-prod.properties +> name: demo-configmap +> kind: Manifest +> ``` + +### Name Customization + +Arrange for the resources to begin with prefix +_prod-_ (since they are meant for the _production_ +environment): + + +``` +cd $DEMO_HOME + +kustomize edit set nameprefix 'prod-' + +cat kustomize.yaml +``` + +`kustomize.yaml` should have updated value of namePrefix field: + +> ``` +> apiVersion: manifest.k8s.io/v1alpha1 +> .... +> namePrefix: prod- +> objectAnnotations: +> note: This is a example annotation +> ``` + +This `namePrefix` directive adds _prod-_ to all +resource names. + + +``` +kustomize build $DEMO_HOME +``` + +The output should contain: +> ``` +> apiVersion: v1 +> data: +> .... +> kind: ConfigMap +> metadata: +> .... +> name: prod-demo-configmap-7746248cmc +> --- +> apiVersion: v1 +> kind: Service +> metadata: +> .... +> name: prod-sbdemo +> spec: +> .... +> --- +> apiVersion: apps/v1beta2 +> kind: Deployment +> metadata: +> .... +> name: prod-sbdemo +> spec: +> selector: +> .... +> ``` + +### Label Customization + +We want resources in production environment to have +certain labels so that we can query them by label +selector. + +`kustomize` does not have `edit set label` command to add +label, but we can edit `kustomize.yaml` file under +`prod` directory and add the production labels under +`objectLabels` fields as highlighted below. + + +``` +sed -i 's/app: helloworld/app: prod/' \ + $DEMO_HOME/kustomize.yaml +``` + +At this point, running `kustomize build` will +generate MySQL configs with name-prefix 'prod-' and +labels `env:prod`. + + +### Download Patch for JVM memory +When a Spring Boot application is deployed in a k8s cluster, the JVM is running inside a container. We want to set memory limit for the container and make sure +the JVM is aware of that limit. In K8s deployment, we can set the resource limits for containers and inject these limits to +some environment variables by downward API. When the container starts to run, it can pick up the environment variables and +set JVM options accordingly. + +Download the patch `memorylimit_patch.yaml`. It contains the memory limits setup. + +``` +cd $DEMO_HOME +wget https://raw.githubusercontent.com/kinflate/example-springboot-instances/master/production/memorylimit_patch.yaml + +cat memorylimit_patch.yaml +``` +The output contains +> ``` +> apiVersion: apps/v1beta2 +> kind: Deployment +> metadata: +> name: sbdemo +> spec: +> template: +> spec: +> containers: +> - name: sbdemo +> resources: +> limits: +> memory: 1250Mi +> requests: +> memory: 1250Mi +> env: +> - name: MEM_TOTAL_MB +> valueFrom: +> resourceFieldRef: +> resource: limits.memory +> ``` + +### Download Patch for health check +We also want to add liveness check and readiness check in the production environment. Spring Boot application +has end points such as `/actuator/health` for this. We can customize the k8s deployment resource to talk to Spring Boot end point. + +Download the patch `healthcheck_patch.yaml`. It contains the liveness probes and readyness probes. + +``` +cd $DEMO_HOME +wget https://raw.githubusercontent.com/kinflate/example-springboot-instances/master/production/healthcheck_patch.yaml + +cat healthcheck_patch.yaml +``` +The output contains +> ``` +> apiVersion: apps/v1beta2 +> kind: Deployment +> metadata: +> name: sbdemo +> spec: +> template: +> spec: +> containers: +> - name: sbdemo +> livenessProbe: +> httpGet: +> path: /actuator/health +> port: 8080 +> initialDelaySeconds: 10 +> periodSeconds: 3 +> readinessProbe: +> initialDelaySeconds: 20 +> periodSeconds: 10 +> httpGet: +> path: /actuator/info +> port: 8080 +> ``` + +### Add patch to Manifest +Currently `kustomize` doesn't provide a command to add a file as a patch, but we can edit the file `kustomize.yaml` to +include this patch. + +``` +mv $DEMO_HOME/kustomize.yaml $DEMO_HOME/tmp.yaml +sed '/patches:$/{N;s/- patch.yaml/- patch.yaml\n- memorylimit_patch.yaml\n- healthcheck_patch.yaml/}' $DEMO_HOME/tmp.yaml >& $DEMO_HOME/kustomize.yaml +``` +`kustomize.yaml` should have patches field: +> ``` +> patches +> - patch.yaml +> - memorylimit_patch.yaml +> - healthcheck_patch.yaml +> ``` + +The output of the following command can now be applied +to the cluster (i.e. piped to `kubectl apply`) to +create the production environment. + + +``` +kustomize build $DEMO_HOME # | kubectl apply -f - +``` diff --git a/demos/tree1.png b/demos/tree1.png new file mode 100644 index 0000000000..a92235187c Binary files /dev/null and b/demos/tree1.png differ diff --git a/demos/tree2.png b/demos/tree2.png new file mode 100644 index 0000000000..275dd899bd Binary files /dev/null and b/demos/tree2.png differ diff --git a/docs/Kube-manifest.yaml b/docs/Kube-manifest.yaml new file mode 100644 index 0000000000..98e59dfb85 --- /dev/null +++ b/docs/Kube-manifest.yaml @@ -0,0 +1,22 @@ +# This is a generated example; do not edit. Rebuild with 'make docs'. + +apiVersion: manifest.k8s.io/v1alpha1 +kind: Manifest +metadata: + name: helloworld +description: helloworld does useful stuff. +namePrefix: some-prefix +# Labels to add to all objects and selectors. +# These labels would also be used to form the selector for apply --prune +# Named differently than “labels” to avoid confusion with metadata for this object +objectLabels: + app: helloworld +objectAnnotations: + note: This is an example annotation +resources: [] +#- service.yaml +#- ../some-dir/ +# There could also be configmaps in Base, which would make these overlays +configMapGenerator: [] +# There could be secrets in Base, if just using a fork/rebase workflow +secretGenerator: [] diff --git a/docs/base.jpg b/docs/base.jpg new file mode 100644 index 0000000000..806d8478c9 Binary files /dev/null and b/docs/base.jpg differ diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000000..e35db30efc --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,261 @@ +# Glossary + +[DAM]: #declarative-application-management +[JSON]: https://www.json.org/ +[Resource]: #resource +[YAML]: http://www.yaml.org/start.html +[application]: #application +[apply]: #apply +[apt]: https://en.wikipedia.org/wiki/APT_(Debian) +[base]: #base +[bases]: #base +[bespoke]: #bespoke-configuration +[kustomize]: #kustomize +[manifest]: #manifest +[off-the-shelf]: #off-the-shelf +[overlay]: #overlay +[overlays]: #overlay +[patch]: #patch +[patches]: #patch +[proposal]: https://github.com/kubernetes/community/pull/1629 +[rebase]: https://git-scm.com/docs/git-rebase +[resource]: #resource +[resources]: #resource +[rpm]: https://en.wikipedia.org/wiki/Rpm_(software) +[target]: #target +[workflow]: workflows.md + +## application + +An _application_ is a group of k8s resources related by +some common purpose, e.g. a load balancer in front of a +webserver backed by a database. +[Resource] labelling, naming and metadata schemes have +historically served to group resources together for +collective operations like _list_ and _remove_. + +This [proposal] describes a new k8s resource called +_application_ to more formally describe this idea and +provide support for application-level operations and +dashboards. + +[kustomize] configures k8s resources, and the proposed +application resource is just another resource. + + +## apply + +The verb _apply_ in the context of k8s refers to a +kubectl command and an in-progress [API +endpoint](https://goo.gl/UbCRuf) for mutating a +cluster. + +One _applies_ a statement of what one wants to a +cluster in the form of a complete resource list. + +The cluster merges this with the previously applied +state and the actual state to arrive at a new desired +state, which the cluster's reconcilation loop attempts +to create. This is the foundation of level-based state +management in k8s. + +## base + +A _base_ is a [target] that some [overlay] modifies. + +Any target, including an overlay, can be a base to +another target. + +A base has no knowledge of the overlays that refer to it. + +A base is usable in isolation, i.e. one should +be able to [apply] a base to a cluster directly. + +## bespoke configuration + +A _bespoke_ configuration is a [manifest] and some +[resources] created and maintained internally by some +organization for their own purposes. + +The [workflow] associated with a _bespoke_ config is +simpler than the workflow associated with an +[off-the-shelf] config, because there's no notion of +periodically capturing someone else's upgrades to the +[off-the-shelf] config. + +## declarative application management + +_Declarative Application Management_ (DAM) is a [set of +ideas](https://goo.gl/T66ZcD) aiming to ease management +of k8s clusters. + + * Works with any configuration, be it bespoke, + off-the-shelf, stateless, stateful, etc. + * Supports common customizations, and creation of + instance variants (dev vs, staging vs. production). + * Exposes and teaches native k8s APIs, rather than + hiding them. + * No friction integration with version control to + support reviews and audit trails. + * Composable with other tools in a unix sense. + * Eschews crossing the line into templating, domain + specific languages, etc., frustrating the other + goals. + +## instance + +An _instance_ is the outcome, in a cluster, of applying +an [overlay] to a [base]. + +> E.g., a _staging_ and _production_ overlay both modify some +> common base to create distinct instances. +> +> The _staging_ instance is the set of resources +> exposed to quality assurance testing, or to some +> external users who'd like to see what the next +> version of production will look like. +> +> The _production_ instance is the set of resources +> exposed to production traffic, and thus may employ +> deployments with a large number of replicas and higher +> cpu and memory requests. + + +## kustomize + +_kustomize_ is a command line tool supporting template-free +customization of declarative configuration targetted to +k8s. + +_Targetted to k8s means_ that kustomize may need some +limited understanding of API resources, k8s concepts +like names, labels, namespaces, etc. and the semantics +of resource patching. + +kustomize is an implementation of [DAM]. + +## manifest + +A _manifest_ is a file called `kustomize.yaml` that +describes a configuration consumable by [kustomize]. + +Here's an [example](Kube-manifest.yaml). + +A manifest contains fields falling into these categories: + + * Immediate customization instructions - + _nameprefix_, _labelprefix_, etc. + * Resource _generators_ for configmaps and secrets. + * References to _external files_ in these categories: + * [resources] - completely specified k8s API objects, + e.g. `deployment.yaml`, `configmap.yaml`, etc. + * [patches] - _partial_ resources that modify full + resources defined in a [base] + (only meaningful in an [overlay]). + * [bases] - path to a directory containing + a [manifest] (only meaningful in an [overlay]). + * (_TBD_) Standard k8s API kind-version fields. + +## off-the-shelf configuration + +An _off-the-shelf_ configuration is a manifest and +resources intentionally published somewhere for others +to use. + +E.g. one might create a github repository like this: + +> ``` +> github.com/username/someapp/ +> kustomize.yaml +> deployment.yaml +> configmap.yaml +> README.md +> ``` + +Someone could then _fork_ this repo (on github) and +_clone_ their fork to their local disk for +customization. + +This clone could act as a [base] for the user's +own [overlays] to do further customization. + +## overlay + +An _overlay_ is a [target] that modifies (and thus +depends on) another target. + +The [manifest] in an overlay refers to (via file path, +URI or other method) to _some other manifest_, known as +its [base]. + +An overlay is unusable without its base. + +An overlay supports the typical notion of a +_development_, _QA_, _staging_ and _production_ +environment instances. + +The configuration of these environments is specified in +individual overlays (one per environment) that all +refer to a common base that holds common configuration. +One configures the cluser like this: + +> ``` +> kustomize inflate someapp/overlays/staging |\ +> kubectl apply -f - +> +> kustomize inflate someapp/overlays/production |\ +> kubectl apply -f - +> ``` + +Usage of the base is implicit (the overlay's manifest +points to the base). + +An overlay may act as a base to another overlay. + +## package + +The word _package_ has no meaning in kustomize, as +kustomize is not to be confused with a package +management tool in the tradition of, say, [apt] or +[rpm]. + +## patch + +A _patch_ is a partially defined k8s resource with a +name that must match a resource already known per +traversal rules built into [kustomize]. + +_Patch_ is a field in the manifest, distinct from +resources, because a patch file looks like a resource +file, but has different semantics. A patch depends on +(modifies) a resource, whereas a resourse has no +dependencies. Since any resource file can be used as a +patch, one cannot reliably distinguish a resource from +a patch just by looking at the file's [YAML]. + +## resource + +A _resource_ is a path to a [YAML] or [JSON] file that +completely defines a functional k8s API object. + +## sub-target / sub-application / sub-package + +A _sub-whatever_ is not a thing. There are only [bases] and [overlays]. + +## target + +The _target_ is the argument to `build`, e.g.: + +> ``` +> kustomize build $target +> ``` + +`$target` must be a path to a directory that +immediately contains a file called +`kustomize.yaml` (i.e. a [manifest]). + +The target contains, or refers to, all the information +needed to create customized resources to send to the +[apply] operation. + +A target is a [base] or an [overlay]. diff --git a/docs/overlay.jpg b/docs/overlay.jpg new file mode 100644 index 0000000000..a8742c6406 Binary files /dev/null and b/docs/overlay.jpg differ diff --git a/docs/workflowBespoke.jpg b/docs/workflowBespoke.jpg new file mode 100644 index 0000000000..9987285763 Binary files /dev/null and b/docs/workflowBespoke.jpg differ diff --git a/docs/workflowOts.jpg b/docs/workflowOts.jpg new file mode 100644 index 0000000000..044b09aac9 Binary files /dev/null and b/docs/workflowOts.jpg differ diff --git a/docs/workflows.md b/docs/workflows.md new file mode 100644 index 0000000000..2c05249c21 --- /dev/null +++ b/docs/workflows.md @@ -0,0 +1,127 @@ +[OTS]: glossary.md#off-the-shelf +[apply]: glossary.md#apply +[applying]: glossary.md#apply +[base]: glossary.md#base +[fork]: https://guides.github.com/activities/forking/ +[instances]: glossary.md#instance +[manifest]: glossary.md#manifest +[off-the-shelf]: glossary.md#off-the-shelf +[overlays]: glossary.md#overlay +[patch]: glossary.md#patch +[patches]: glossary.md#patch +[rebase]: https://git-scm.com/docs/git-rebase +[resources]: glossary.md#resources +[workflowBespoke]: workflowBespoke.jpg +[workflowOts]: workflowOts.jpg + +# workflows + +A _workflow_ is the sequence of steps one takes to +use and maintain a configuration. + +## Bespoke configuration + +In this workflow, all configuration files are owned by +the user. No content is incorporated from version +control repositories owned by others. + +![bespoke config workflow image][workflowBespoke] + +#### 1) create a directory in version control + +> ``` +> git init ~/ldap +> ``` + +#### 2) create a [base] + +> ``` +> mkdir -p ~/ldap/base +> ``` + +In this directory, create and commit a [manifest] +and a set of [resources]. + +#### 3) create [overlays] + +> ``` +> mkdir -p ~/ldap/overlays/staging +> mkdir -p ~/ldap/overlays/production +> ``` + +Each of these directories needs a [manifest] +and one or more [patches]. + +The _staging_ directory might get a patch +that turns on an experiment flag in a configmap. + +The _production_ directory might get a patch +that increases the replica count in a deployment +specified in the base. + +#### 4) bring up [instances] + +Run kustomize, and pipe the output to [apply]. + +> ``` +> kustomize ~/ldap/overlays/staging | kubectl apply -f - +> kustomize ~/ldap/overlays/production | kubectl apply -f - +> ``` + + +## Off-the-shelf configuration + +In this workflow, all files are owned by the user and +maintained in a repository under their control, but +they are based on an [off-the-shelf] configuration that +is periodically consulted for updates. + + +![off-the-shelf config workflow image][workflowOts] + +#### 1) find and [fork] an [OTS] config + +#### 2) clone it as your [base] + +The [base] directory is maintained in a repo whose +upstream is an [OTS] configuration, in this case +https://github.com/kinflate/ldap. + +> ``` +> mkdir ~/ldap +> git clone https://github.com/$USER/ldap ~/ldap/base +> cd ~/ldap/base +> git remote add upstream git@github.com:kustomize/ldap +> ``` + +#### 3) create [overlays] + +As in the bespoke case above, create and populate +an _overlays_ directory. + +The [overlays] are siblings to each other and to the +[base] they depend on. + +> ``` +> mkdir -p ~/ldap/overlays/staging +> mkdir -p ~/ldap/overlays/production +> ``` + + +#### 4) bring up instances + +> ``` +> kustomize ~/ldap/overlays/staging | kubectl apply -f - +> kustomize ~/ldap/overlays/production | kubectl apply -f - +> ``` + +#### 5) (optionally) capture changes from upstream + +The user can optionally [rebase] their [base] to +capture changes made in the upstream repository. + +> ``` +> cd ~/ldap/base +> git fetch upstream +> git rebase upstream/master +> ``` diff --git a/kustomize.go b/kustomize.go new file mode 100644 index 0000000000..7940c4dd09 --- /dev/null +++ b/kustomize.go @@ -0,0 +1,34 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/golang/glog" + + "k8s.io/kubectl/pkg/kustomize/commands" +) + +func main() { + defer glog.Flush() + + if err := commands.NewDefaultCommand().Execute(); err != nil { + os.Exit(1) + } + os.Exit(0) +} diff --git a/test/main.sh b/test/main.sh new file mode 100755 index 0000000000..334ed80810 --- /dev/null +++ b/test/main.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function exit_with { + local msg=$1 + echo >&2 ${msg} + exit 1 +} + +base_dir="$( cd "$(dirname "$0")/../../.." && pwd )" +cd "$base_dir" || { + echo "Cannot cd to '$base_dir'. Aborting." >&2 + exit 1 +} + +# Install kustomize to $GOPATH/bin and export PATH +go install ./cmd/kustomize || { exit_with "Failed to install kustomize"; } +export PATH=$GOPATH/bin:$PATH + +home=`pwd` +example_dir="some/default/dir/for/examples" +if [ $# -eq 1 ]; then + example_dir=$1 +fi +if [ ! -d ${example_dir} ]; then + exit_with "directory ${example_dir} doesn't exist" +fi + +if [ -x "${example_dir}/tests/test.sh" ]; then + ${example_dir}/tests/test.sh ${example_dir} + if [ $? -eq 0 ]; then + echo "testing ${example_dir} passed." + else + exit_with "testing ${example_dir} failed." + fi +fi \ No newline at end of file diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000000..36aa6b52bd --- /dev/null +++ b/version/version.go @@ -0,0 +1,67 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" +) + +var ( + kustomizeVersion = "unknown" + goos = "unknown" + goarch = "unknown" + gitCommit = "$Format:%H$" // sha1 from git, output of $(git rev-parse HEAD) + + buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') +) + +type Version struct { + KustomizeVersion string `json:"kustomizeVersion"` + GitCommit string `json:"gitCommit"` + BuildDate string `json:"buildDate"` + GoOs string `json:"goOs"` + GoArch string `json:"goArch"` +} + +func GetVersion() Version { + return Version{ + kustomizeVersion, + gitCommit, + buildDate, + goos, + goarch, + } +} + +func (v Version) Print(w io.Writer) { + fmt.Fprintf(w, "Version: %+v\n", v) +} + +// NewCmdVersion makes version command. +func NewCmdVersion(w io.Writer) *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Prints the kustomize version", + Example: `kustomize version`, + Run: func(cmd *cobra.Command, args []string) { + GetVersion().Print(w) + }, + } +}