Skip to content

Commit

Permalink
feat: add Tilt support for local development
Browse files Browse the repository at this point in the history 
This will allow for more rapid development as tilt supports
hot reload on code changes.
  • Loading branch information
@joekr
joekr committed Jun 8, 2022
1 parent b5cca41 commit d4c285a
Show file tree
Hide file tree
Showing 13 changed files with 526 additions and 3 deletions.
59 changes: 58 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
TAG ?= dev
ARCH ?= amd64
ALL_ARCH = amd64 arm64

TOOLS_DIR := hack/tools
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)

GINKGO_VER := v1.16.5
Expand Down Expand Up @@ -60,6 +60,21 @@ EXP_MACHINE_POOL ?= false
# Set build time variables including version details
LDFLAGS := $(shell source ./hack/version.sh; version::ldflags)

# curl retries
CURL_RETRIES=3

ENVSUBST_VER := v2.0.0-20210730161058-179042472c46
ENVSUBST_BIN := envsubst
ENVSUBST := $(TOOLS_BIN_DIR)/$(ENVSUBST_BIN)-$(ENVSUBST_VER)

KUSTOMIZE_VER := v4.5.2
KUSTOMIZE_BIN := kustomize
KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)

KUBECTL_VER := v1.22.9
KUBECTL_BIN := kubectl
KUBECTL := $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)-$(KUBECTL_VER)

# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
Expand Down Expand Up @@ -326,3 +341,45 @@ release-metadata: $(RELEASE_DIR)
.PHONY: clean-release
clean-release:
rm -rf $(RELEASE_DIR)

GO_INSTALL = ./scripts/go_install.sh
GOOS := $(shell go env GOOS)
GOARCH := $(shell go env GOARCH)

.PHONY: install-tools # populate hack/tools/bin
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL)

envsubst: $(ENVSUBST) ## Build a local copy of envsubst.
kustomize: $(KUSTOMIZE) ## Build a local copy of kustomize.
kubectl: $(KUBECTL) ## Build a local copy of kubectl.

$(ENVSUBST): ## Build envsubst from tools folder.
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/drone/envsubst/v2/cmd/envsubst $(ENVSUBST_BIN) $(ENVSUBST_VER)

$(KUSTOMIZE): ## Build kustomize from tools folder.
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kustomize/kustomize/v4 $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER)

$(KUBECTL): ## Build kubectl from tools folder.
mkdir -p $(TOOLS_BIN_DIR)
rm -f "$(TOOLS_BIN_DIR)/$(KUBECTL_BIN)*"
curl --retry $(CURL_RETRIES) -fsL https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VER)/bin/$(GOOS)/$(GOARCH)/kubectl -o $(KUBECTL)
ln -sf $(KUBECTL) $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)
chmod +x $(KUBECTL) $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)

.PHONY: $(ENVSUBST_BIN)
$(ENVSUBST_BIN): $(ENVSUBST)

.PHONY: $(KUBECTL_BIN)
$(KUBECTL_BIN): $(KUBECTL)

## --------------------------------------
## Tilt / Kind
## --------------------------------------

.PHONY: kind-create
kind-create: $(KUBECTL) ## Create kind cluster if needed.
./scripts/kind-with-registry.sh

.PHONY: tilt-up
tilt-up: install-tools kind-create ## Start tilt and build kind cluster if needed.
EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true tilt up
215 changes: 215 additions & 0 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@

envsubst_cmd = "./hack/tools/bin/envsubst"
kubectl_cmd = "./hack/tools/bin/kubectl"
helm_cmd = "./hack/tools/bin/helm"
tools_bin = "./hack/tools/bin"

update_settings(k8s_upsert_timeout_secs = 60) # on first tilt up, often can take longer than 30 seconds

#Add tools to path
os.putenv("PATH", os.getenv("PATH") + ":" + tools_bin)

keys = ["OCI_TENANCY_ID", "OCI_USER_ID", "OCI_CREDENTIALS_FINGERPRINT", "OCI_REGION", "OCI_CREDENTIALS_KEY_PATH"]

# set defaults
settings = {
"allowed_contexts": [
"kind-capoci",
],
"deploy_cert_manager": True,
"preload_images_for_kind": True,
"kind_cluster_name": "capoci",
"capi_version": "v1.1.3",
"cert_manager_version": "v1.1.0",
"kubernetes_version": "v1.22.9",
}

# global settings
settings.update(read_json(
"tilt-settings.json",
default = {},
))

if settings.get("trigger_mode") == "manual":
trigger_mode(TRIGGER_MODE_MANUAL)

if "allowed_contexts" in settings:
allow_k8s_contexts(settings.get("allowed_contexts"))

if "default_registry" in settings:
default_registry(settings.get("default_registry"))

tilt_helper_dockerfile_header = """
# Tilt image
FROM golang:1.17 as tilt-helper
# Support live reloading with Tilt
RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh && \
wget --output-document /start.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/start.sh && \
chmod +x /start.sh && chmod +x /restart.sh
"""

tilt_dockerfile_header = """
FROM gcr.io/distroless/base:debug as tilt
WORKDIR /
COPY --from=tilt-helper /start.sh .
COPY --from=tilt-helper /restart.sh .
COPY manager .
"""

def validate_auth():
substitutions = settings.get("kustomize_substitutions", {})
os.environ.update(substitutions)
for sub in substitutions:
if sub[-4:] == "_B64":
os.environ[sub[:-4]] = base64_decode(os.environ[sub])
print("{} was not specified in tilt-settings.json, attempting to load {}".format(base64_decode(os.environ[sub]), sub))
missing = [k for k in keys if not os.environ.get(k)]
if missing:
fail("missing kustomize_substitutions keys {} in tilt-setting.json".format(missing))

# Users may define their own Tilt customizations in tilt.d. This directory is excluded from git and these files will
# not be checked in to version control.
def include_user_tilt_files():
user_tiltfiles = listdir("tilt.d")
for f in user_tiltfiles:
include(f)

# deploy CAPI
def deploy_capi():
version = settings.get("capi_version")
capi_uri = "https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(version)
cmd = "curl -sSL {} | {} | {} apply -f -".format(capi_uri, envsubst_cmd, kubectl_cmd)
local(cmd, quiet = False)
if settings.get("extra_args"):
extra_args = settings.get("extra_args")
if extra_args.get("core"):
core_extra_args = extra_args.get("core")
if core_extra_args:
for namespace in ["capi-system", "capi-webhook-system"]:
patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args)
if extra_args.get("kubeadm-bootstrap"):
kb_extra_args = extra_args.get("kubeadm-bootstrap")
if kb_extra_args:
patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args)

def patch_args_with_extra_args(namespace, name, extra_args):
args_str = str(local("{} get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[1].args}}".format(kubectl_cmd, name, namespace)))
args_to_add = [arg for arg in extra_args if arg not in args_str]
if args_to_add:
args = args_str[1:-1].split()
args.extend(args_to_add)
patch = [{
"op": "replace",
"path": "/spec/template/spec/containers/1/args",
"value": args,
}]
local("{} patch deployment {} -n {} --type json -p='{}'".format(kubectl_cmd, name, namespace, str(encode_json(patch)).replace("\n", "")))

# Build CAPOCI and add feature gates
def capoci():
# Apply the kustomized yaml for this provider
yaml = str(kustomizesub("./config/default"))

# add extra_args if they are defined
if settings.get("extra_args"):
oci_extra_args = settings.get("extra_args").get("oci")
if oci_extra_args:
yaml_dict = decode_yaml_stream(yaml)
append_arg_for_container_in_deployment(yaml_dict, "capoci-controller-manager", "capoci-system", "cluster-api-oci-controller", oci_extra_args)
yaml = str(encode_yaml_stream(yaml_dict))
yaml = fixup_yaml_empty_arrays(yaml)

# Set up a local_resource build of the provider's manager binary.
local_resource(
"manager",
cmd = 'mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags \'-extldflags "-static"\' -o .tiltbuild/manager',
deps = ["api", "cloud", "config", "controllers", "exp", "feature", "pkg", "go.mod", "go.sum", "main.go", "auth-config.yaml"],
labels = ["cluster-api"],
)

dockerfile_contents = "\n".join([
tilt_helper_dockerfile_header,
tilt_dockerfile_header,
])

entrypoint = ["sh", "/start.sh", "/manager"]
extra_args = settings.get("extra_args")
if extra_args:
entrypoint.extend(extra_args)

# Set up an image build for the provider. The live update configuration syncs the output from the local_resource
# build into the container.
docker_build(
ref = "ghcr.io/oracle/cluster-api-oci-controller-amd64:dev",
context = "./.tiltbuild/",
dockerfile_contents = dockerfile_contents,
target = "tilt",
entrypoint = entrypoint,
only = "manager",
live_update = [
sync(".tiltbuild/manager", "/manager"),
run("sh /restart.sh"),
],
ignore = ["templates"],
network = "host",
)

#secret_settings(disable_scrub=True)
k8s_yaml(blob(yaml))


def append_arg_for_container_in_deployment(yaml_stream, name, namespace, contains_image_name, args):
for item in yaml_stream:
if item["kind"] == "Deployment" and item.get("metadata").get("name") == name and item.get("metadata").get("namespace") == namespace:
containers = item.get("spec").get("template").get("spec").get("containers")
for container in containers:
if contains_image_name in container.get("image"):
container.get("args").extend(args)

def fixup_yaml_empty_arrays(yaml_str):
yaml_str = yaml_str.replace("conditions: null", "conditions: []")
return yaml_str.replace("storedVersions: null", "storedVersions: []")

def waitforsystem():
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-bootstrap-system")
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system")
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system")

def base64_encode(to_encode):
encode_blob = local("echo '{}' | tr -d '\n' | base64 - | tr -d '\n'".format(to_encode), quiet = True, echo_off = True)
return str(encode_blob)

def base64_encode_file(path_to_encode):
encode_blob = local("cat {} | tr -d '\n' | base64 - | tr -d '\n'".format(path_to_encode), quiet = True)
return str(encode_blob)

def read_file_from_path(path_to_read):
str_blob = local("cat {} | tr -d '\n'".format(path_to_read), quiet = True)
return str(str_blob)

def base64_decode(to_decode):
decode_blob = local("echo '{}' | base64 --decode -".format(to_decode), quiet = True, echo_off = True)
return str(decode_blob)

def kustomizesub(folder):
yaml = local('hack/kustomize-sub.sh {}'.format(folder), quiet=True)
return yaml

##############################
# Actual work happens here
##############################

validate_auth()

include_user_tilt_files()

load("ext://cert_manager", "deploy_cert_manager")

if settings.get("deploy_cert_manager"):
deploy_cert_manager()

deploy_capi()

capoci()

waitforsystem()
2 changes: 0 additions & 2 deletions config/manager/manager.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ spec:
labels:
control-plane: controller-manager
spec:
securityContext:
runAsNonRoot: true
containers:
- command:
- /manager
Expand Down
22 changes: 22 additions & 0 deletions hack/kustomize-sub.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

#!/bin/bash
# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

root=$(dirname "${BASH_SOURCE[0]}")
$root/tools/bin/kustomize build $1 | $root/tools/bin/envsubst
63 changes: 63 additions & 0 deletions local-dev.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Local Development

## Tilt

### Tilt Requirements

Install [Tilt](https://docs.tilt.dev/install.html):

- `brew install tilt-dev/tap/tilt` on macOS or Linux.
- `scoop bucket add tilt-dev https://github.com/tilt-dev/scoop-bucket` & `scoop install tilt` on Windows
- for alternatives you can follow the installation instruction for [macOS](https://docs.tilt.dev/install.html#macos), [Linux](https://docs.tilt.dev/install.html#linux) or [Windows](https://docs.tilt.dev/install.html#windows)

After the installation is done, verify that you have installed it correctly with: `tilt version`.

Install Helm:

- brew install helm on MacOS
- `choco install kubernetes-helm` on Windows
- [Install Instruction](https://helm.sh/docs/intro/install/#from-source-linux-macos) on Linux

You would require installation of Helm for successfully setting up Tilt.

### Using Tilt

From the root of the CAPOCI repository and after configuring the [environment
variables](https://oracle.github.io/cluster-api-provider-oci/gs/install-cluster-api.html#configure-authentication),
you can run the following to generate your `tilt-settings.json` file:

```
cat <<EOF > tilt-settings.json
{
"kustomize_substitutions": {
"OCI_TENANCY_ID_B64": "$(echo "${OCI_TENANCY_ID}" | tr -d '\n' | base64 | tr -d '\n')",
"OCI_CREDENTIALS_FINGERPRINT_B64": "$(echo "${OCI_CREDENTIALS_FINGERPRINT}" | tr -d '\n' | base64 | tr -d '\n')",
"OCI_USER_ID_B64": "$(echo "${OCI_USER_ID}" | tr -d '\n' | base64 | tr -d '\n')",
"OCI_REGION_B64": "$(echo "${OCI_REGION}" | tr -d '\n' | base64 | tr -d '\n')",
"OCI_CREDENTIALS_KEY_B64": "$(echo "${OCI_CREDENTIALS_KEY_PATH}" | tr -d '\n' | base64 | tr -d '\n')"
}
}
EOF
```

To build a kind cluster and start Tilt, just run:

```
make tilt-up
```

Once your kind management cluster is up and running, you can deploy a [workload cluster](https://oracle.github.io/cluster-api-provider-oci/gs/create-workload-cluster.html).

To tear down the kind cluster built by the command above you will need to
delete your workload cluster:

```
kubectl delete cluster <CLUSTER_NAME>
```

then delete your local management cluster:

```
kind delete cluster
```

0 comments on commit d4c285a

Please sign in to comment.