Skip to content

Commit

Permalink
rollouts: scripts and dev guide for running with kind (#3888)
Browse files Browse the repository at this point in the history
  • Loading branch information
natasha41575 committed Mar 16, 2023
1 parent 879298b commit 0a89c06
Show file tree
Hide file tree
Showing 8 changed files with 273 additions and 14 deletions.
10 changes: 9 additions & 1 deletion rollouts/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,14 @@ license: ## Add licenses.
test: manifests generate fmt vet envtest ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out

.PHONY: run-in-kind
run-in-kind: manifests kustomize ## Run the rollouts-controller-manager in a kind cluster named rollouts-management-cluster
IMG=${IMG} KUSTOMIZE=${KUSTOMIZE} ./scripts/run-in-kind.sh

.PHONY: run-target-in-kind ## Create a target kind cluster with CS installed
run-target-in-kind:
./scripts/run-target-in-kind.sh

##@ Build

.PHONY: build
Expand Down Expand Up @@ -139,7 +147,7 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
$(KUSTOMIZE) build config/default | kubectl apply -f -

.PHONY: dry-run
dry-run: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
dry-run: manifests kustomize ## Build the manifests that would be deployed to the k8s cluster without applying them.
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/default > bin/all.yaml

Expand Down
1 change: 1 addition & 0 deletions rollouts/config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,5 +112,6 @@ spec:
requests:
cpu: 10m
memory: 64Mi
imagePullPolicy: Always
serviceAccountName: controller-manager
terminationGracePeriodSeconds: 10
8 changes: 8 additions & 0 deletions rollouts/config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@ metadata:
creationTimestamp: null
name: manager-role
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- container.cnrm.cloud.google.com
resources:
Expand Down
1 change: 1 addition & 0 deletions rollouts/controllers/rollout_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ type RolloutReconciler struct {
packageDiscoveryCache map[types.NamespacedName]*packagediscovery.PackageDiscovery
}

//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
//+kubebuilder:rbac:groups=container.cnrm.cloud.google.com,resources=containerclusters,verbs=get;list;watch
//+kubebuilder:rbac:groups=gitops.kpt.dev,resources=rollouts,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=gitops.kpt.dev,resources=rollouts/status,verbs=get;update;patch
Expand Down
2 changes: 1 addition & 1 deletion rollouts/docs/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ make docker-push
# IMG - image name and tag

# Example:
gcr.io/<your-project>/rollouts-controller:v0.0.x make docker-push
IMG=gcr.io/<your-project>/rollouts-controller:v0.0.x make docker-push
```

## Running Locally
Expand Down
167 changes: 155 additions & 12 deletions rollouts/docs/running-locally.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,152 @@ To run Rollouts locally, you will need:
confirmed to work on Linux)
* [go 1.19](https://go.dev/dl/) or newer
* `make`
* Access to GKE clusters.
* Either access to GKE clusters, or Kind.

## Cluster Setup
This doc will go through:

- [Running in kind](#running-in-kind): How to run the controller and target clusters as Kind clusters.
- [Running locally with a KCC Cluster](#running-the-controller-locally-with-a-kcc-management-cluster): How to run the controller locally while connected to a KCC management cluster and target clusters.

There are also sample Rollout objects for each.

---

## Running in Kind

### Creating a management cluster
To spin up a admin Kind cluster with the rollouts controller, run:

```sh
make run-in-kind`
```

This will create a new kind cluster for you called `rollouts-management-cluster` with the rollouts
controller manager running and relevant CRDs installed. It will store the kubeconfig for the
new kind cluster in `~/.kube/admin-cluster`.

Verify that the admin cluster has been created correctly with your kubeconfig setup:

```sh
KUBECONFIG=~/.kube/admin-cluster kubectl get nodes
NAME STATUS ROLES AGE VERSION
rollouts-management-cluster-control-plane Ready control-plane 57s v1.25.3
```

Verify the CRDs are installed:

```sh
$ KUBECONFIG=~/.kube/admin-cluster kubectl api-resources | grep gitops
progressiverolloutstrategies gitops.kpt.dev/v1alpha1 true ProgressiveRolloutStrategy
remotesyncs gitops.kpt.dev/v1alpha1 true RemoteSync
rollouts gitops.kpt.dev/v1alpha1 true Rollout
```

Verify the controller is running:

```sh
$ KUBECONFIG=~/.kube/admin-cluster kubectl get pods -nrollouts-system
NAME READY STATUS RESTARTS AGE
rollouts-controller-manager-7f556b8667-6kzbl 2/2 Running 0 20s
```

### Restarting the management controller

If you make code changes, all you have to do is rerun `make run-in-kind`.

### Creating the target clusters

To create a Kind target cluster, run:

```sh
make run-target-in-kind
```

This will spin up a new Kind cluster and install Config Sync to it. It will also create
a ConfigMap representation of the Kind cluster in the `kind-clusters` namespace, and the ConfigMap will have
a sample label `location: example` as well as the kubeconfig for the new Kind cluster. It will also store the new Kind
cluster's kubeconfig in `~/.kube/$NAME`.
The default name for the cluster `rollouts-target`, and default Config Sync version installed is `v1.14.2`.
Verify that the target cluster has been created correctly with your kubeconfig setup:
```sh
KUBECONFIG=~/.kube/rollouts-target kubectl get nodes
NAME STATUS ROLES AGE VERSION
rollouts-target-control-plane Ready control-plane 24m v1.25.3
```
Verify that Config Sync was installed:
```sh
KUBECONFIG=~/.kube/rollouts-target kubectl api-resources | grep configsync
reposyncs configsync.gke.io/v1beta1 true RepoSync
rootsyncs configsync.gke.io/v1beta1 true RootSync
```
You can optionally specify a name for the target cluster, and a version of Config Sync that you want installed:
```sh
NAME=target CS_VERSION=vX.Y.Z make run-target-in-kind
```
### Creating a Rollout object
Here is an example of a simple `Rollout` object which uses Kind clusters:
```yaml
apiVersion: gitops.kpt.dev/v1alpha1
kind: Rollout
metadata:
name: sample
spec:
description: sample
clusters:
sourceType: Kind
kind:
namespace: kind-clusters
packages:
sourceType: GitHub
github:
selector:
org: droot
repo: store
directory: namespaces
revision: main
targets:
selector:
matchLabels:
location: example
syncTemplate:
type: RootSync
packageToTargetMatcher:
type: AllClusters
strategy:
type: RollingUpdate
rollingUpdate:
maxConcurrent: 2
```
Apply this to your management cluster with `kubectl apply -f`. View the created Rollout, RemoteSyncs, and RootSync objects to verify that the controller is running properly:
```sh
# see the rollouts object
KUBECONFIG=~/.kube/admin-cluster kubectl get rollouts sample
# see the remotesync objects that the rollouts controller created
KUBECONFIG=~/.kube/admin-cluster kubectl get remotesyncs
# see the rootsync object that the remotesync controller created
KUBECONFIG=~/.kube/rollouts-target kubectl get rootsyncs -nconfig-management-system
```
Deleting the Rollout object should likewise delete the associated RemoteSync and Rootsync objects. You can
look at the controller logs to verify that the various Remotesync/Rootsync objects are being created, updated,
or deleted.
---
## Running the controller locally with a KCC Management cluster.
### Creating a management cluster with KCC running
First, you must create a config-controller cluster. You can follow the instructions on the
Expand All @@ -23,9 +166,9 @@ Make sure your kubeconfig is connected to this cluster:
KUBECONFIG=~/.kube/admin-cluster gcloud container clusters get-credentials <your cluster> --region <your region> --project <your project>
```
### Provisioning child clusters
The next step will be to provision new child clusters. This example names the child cluster `gke-n`;
you can replace `gke-n` with whatever name you want for your child cluster.
### Provisioning target clusters
The next step will be to provision new target clusters. This example names the target cluster `gke-n`;
you can replace `gke-n` with whatever name you want for your target cluster.
```sh
# first step is to create a deployable instance of the package
Expand All @@ -40,7 +183,7 @@ KUBECONFIG=~/.kube/admin-cluster kpt live init gke-n
KUBECONFIG=~/.kube/admin-cluster kpt live apply gke-n
```
You can repeat the above steps to create as many child clusters as you want.
You can repeat the above steps to create as many target clusters as you want.
### Setting up kubeconfig
Next, we will configure kubeconfig to be able to talk to each cluster individually.
Expand All @@ -56,8 +199,8 @@ KUBECONFIG=~/.kube/gke-n kubectl get pods -n kube-system
```
### Set up Config Sync
Rollouts requires a git syncer installed on child clusters. Currently, only Config Sync is supported.
To install Config Sync on your child clusters, follow these steps.
Rollouts requires a git syncer installed on target clusters. Currently, only Config Sync is supported.
To install Config Sync on your target clusters, follow these steps.
First, set the release version of Config Sync that you would like to install:
Expand All @@ -81,7 +224,7 @@ If you wish to install Config Sync from source instead of using a released versi
the [Config Sync installation guide](https://github.com/GoogleContainerTools/kpt-config-sync/blob/main/docs/installation.md).
## Running Rollouts locally
### Running Rollouts controller locally
Clone this repository into `${GOPATH}/src/github.com/GoogleContainerTools/kpt`.
Expand Down Expand Up @@ -124,9 +267,9 @@ Now you are ready to run the controller locally:
KUBECONFIG=~/.kube/admin-cluster go run main.go
```
## Creating a Rollout object
### Creating a Rollout object
Here is an example of a simple `Rollout` object:
Here is an example of a simple `Rollout` object which uses KCC:
```yaml
apiVersion: gitops.kpt.dev/v1alpha1
Expand Down Expand Up @@ -176,7 +319,7 @@ Deleting the Rollout object should likewise delete the associated RemoteSync and
look at the controller logs to verify that the various Remotesync/Rootsync objects are being created, updated,
or deleted.
## Restarting Rollouts
### Restarting the local controller
If you make code changes, all you have to do is stop the controller (ctrl+C in the terminal where it is running),
and rerun `go run main.go`.
Expand Down
36 changes: 36 additions & 0 deletions rollouts/scripts/run-in-kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#! /usr/bin/env bash
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

mkdir -p ~/.kube
KUBECONFIG=~/.kube/admin-cluster

# create kind cluster
kind delete cluster --name=rollouts-management-cluster --kubeconfig $KUBECONFIG
kind create cluster --name=rollouts-management-cluster --kubeconfig $KUBECONFIG

# install crds
KUBECONFIG=$KUBECONFIG make install
KUBECONFIG=$KUBECONFIG kubectl apply -f ./manifests/crds/containercluster.yaml

# build and set image
IMG=$IMG make docker-build
kind load docker-image $IMG --name=rollouts-management-cluster
(cd config/manager && $KUSTOMIZE edit set image controller=$IMG)

# deploy
$KUSTOMIZE build ./config/default | sed --expression='s/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/g' | KUBECONFIG=$KUBECONFIG kubectl apply -f -

# wait for controller to be ready
KUBECONFIG=$KUBECONFIG kubectl rollout status deployment rollouts-controller-manager --namespace rollouts-system
62 changes: 62 additions & 0 deletions rollouts/scripts/run-target-in-kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#! /usr/bin/env bash
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

KUBECONFIG=/tmp/kubeconfig.yaml
KIND_CONFIG=/tmp/kind-cluster.yaml
MGMTKUBECONFIG=~/.kube/admin-cluster

DEFAULT_NAME=rollouts-target
DEFAULT_NAMESPACE=kind-clusters

# TODO: fetch the latest version of config sync instead of hardcoding it here
DEFAULT_CS_VERSION="v1.14.2"

# set default values
if [ -z "$NAME" ]
then
NAME=$DEFAULT_NAME
fi

if [ -z "$NAMESPACE" ]
then
NAMESPACE=$DEFAULT_NAMESPACE
fi

if [ -z "$CS_VERSION" ]
then
CS_VERSION=$DEFAULT_CS_VERSION
fi

# create the kind cluster and store the kubeconfig
cat <<EOF > $KIND_CONFIG
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
apiServerAddress: "$(hostname -I | awk '{print $1}')"
EOF

kind create cluster --config $KIND_CONFIG --kubeconfig $KUBECONFIG --name $NAME
kind get kubeconfig --name $NAME > $KUBECONFIG

# create the kind-clusters namespace and put the ConfigMap in it
KUBECONFIG=$MGMTKUBECONFIG kubectl create namespace $NAMESPACE
KUBECONFIG=$MGMTKUBECONFIG kubectl create configmap $NAME -n $NAMESPACE --from-file=kubeconfig.yaml=$KUBECONFIG
KUBECONFIG=$MGMTKUBECONFIG kubectl label configmap $NAME -n $NAMESPACE location=example

mkdir -p ~/.kube
cp $KUBECONFIG ~/.kube/$NAME

# install config sync
KUBECONFIG=~/.kube/$NAME kubectl apply -f "https://github.com/GoogleContainerTools/kpt-config-sync/releases/download/${CS_VERSION}/config-sync-manifest.yaml"

0 comments on commit 0a89c06

Please sign in to comment.