From 5614cd53904eb0cbe3e5d8c02590a5d5a3aa30ef Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Mon, 17 Jul 2023 18:39:13 +0800 Subject: [PATCH] Add release manifests stage-fast.yaml and kwokctl.yaml --- Makefile | 4 +- hack/manifests.sh | 9 +- kustomize/stage/fast/kustomization.yaml | 5 + kustomize/stage/node/fast/README.md | 7 ++ kustomize/stage/node/fast/kustomization.yaml | 4 + .../stage/node/fast/node-initialize.yaml | 78 ++++++++++++++ kustomize/stage/node/heartbeat/README.md | 6 ++ .../stage/node/heartbeat/kustomization.yaml | 4 + .../stage/node/heartbeat/node-heartbeat.yaml | 35 ++++++ kustomize/stage/pod/fast/README.md | 15 +++ kustomize/stage/pod/fast/kustomization.yaml | 6 ++ kustomize/stage/pod/fast/pod-complete.yaml | 40 +++++++ kustomize/stage/pod/fast/pod-delete.yaml | 16 +++ kustomize/stage/pod/fast/pod-ready.yaml | 63 +++++++++++ kustomize/stage/pod/general/README.md | 4 + .../stage/pod/general/kustomization.yaml | 10 ++ kustomize/stage/pod/general/pod-complete.yaml | 49 +++++++++ kustomize/stage/pod/general/pod-create.yaml | 101 ++++++++++++++++++ kustomize/stage/pod/general/pod-delete.yaml | 21 ++++ .../general/pod-init-container-completed.yaml | 61 +++++++++++ .../general/pod-init-container-running.yaml | 42 ++++++++ kustomize/stage/pod/general/pod-ready.yaml | 59 ++++++++++ .../pod/general/pod-remove-finalizer.yaml | 28 +++++ .../en/docs/user/stages-configuration.md | 47 +------- test/release/build.test.sh | 16 +-- 25 files changed, 673 insertions(+), 57 deletions(-) create mode 100644 kustomize/stage/fast/kustomization.yaml create mode 100644 kustomize/stage/node/fast/README.md create mode 100644 kustomize/stage/node/fast/kustomization.yaml create mode 100644 kustomize/stage/node/fast/node-initialize.yaml create mode 100644 kustomize/stage/node/heartbeat/README.md create mode 100644 kustomize/stage/node/heartbeat/kustomization.yaml create mode 100644 kustomize/stage/node/heartbeat/node-heartbeat.yaml create mode 100644 kustomize/stage/pod/fast/README.md create mode 100644 kustomize/stage/pod/fast/kustomization.yaml create mode 100644 kustomize/stage/pod/fast/pod-complete.yaml create mode 100644 kustomize/stage/pod/fast/pod-delete.yaml create mode 100644 kustomize/stage/pod/fast/pod-ready.yaml create mode 100644 kustomize/stage/pod/general/README.md create mode 100644 kustomize/stage/pod/general/kustomization.yaml create mode 100644 kustomize/stage/pod/general/pod-complete.yaml create mode 100644 kustomize/stage/pod/general/pod-create.yaml create mode 100644 kustomize/stage/pod/general/pod-delete.yaml create mode 100644 kustomize/stage/pod/general/pod-init-container-completed.yaml create mode 100644 kustomize/stage/pod/general/pod-init-container-running.yaml create mode 100644 kustomize/stage/pod/general/pod-ready.yaml create mode 100644 kustomize/stage/pod/general/pod-remove-finalizer.yaml diff --git a/Makefile b/Makefile index f8a847a8f..38e839705 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,8 @@ IMAGE_PLATFORMS ?= linux/amd64 linux/arm64 BINARY_PLATFORMS ?= linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64 +MANIFESTS ?= kwok kwokctl stage/fast + BUILDER ?= docker DOCKER_CLI_EXPERIMENTAL ?= enabled @@ -219,7 +221,7 @@ cross-cluster-image: .PHONY: manifests manifests: @./hack/manifests.sh \ - --kustomize=kwok \ + $(addprefix --kustomize=, $(MANIFESTS)) \ --bucket=${BUCKET} \ --gh-release=${GH_RELEASE} \ --image-prefix=${IMAGE_PREFIX} \ diff --git a/hack/manifests.sh b/hack/manifests.sh index 42817bb88..52990acdb 100755 --- a/hack/manifests.sh +++ b/hack/manifests.sh @@ -106,24 +106,25 @@ function main() { dry_run mkdir -p "./artifacts" dry_run cp -r "./kustomize" "./artifacts/" for kustomize in "${KUSTOMIZES[@]}"; do + outfile="${kustomize//\//-}" dry_run cd "./artifacts/kustomize/${kustomize}" dry_run kustomize edit set image "${DEFAULT_IMAGE_PREFIX}/kwok=${IMAGE_PREFIX}/kwok:${VERSION}" - dry_run kustomize build "." -o "../../${kustomize}.yaml" dry_run cd - - dry_run rm -r "./artifacts/kustomize" + dry_run kustomize build "./artifacts/kustomize/${kustomize}" -o "./artifacts/${outfile}.yaml" if [[ "${PUSH}" == "true" ]]; then if [[ "${BUCKET}" != "" ]]; then prefix="${BUCKET}/releases/" if [[ "${STAGING_PREFIX}" != "" ]]; then prefix="${BUCKET}/releases/${STAGING_PREFIX}-" fi - dry_run gsutil cp -P "./artifacts/${kustomize}.yaml" "${prefix}${VERSION}/manifests/${kustomize}.yaml" + dry_run gsutil cp -P "./artifacts/${outfile}.yaml" "${prefix}${VERSION}/manifests/${outfile}.yaml" fi if [[ "${GH_RELEASE}" != "" ]]; then - dry_run gh -R "${GH_RELEASE}" release upload "${VERSION}" "./artifacts/${kustomize}.yaml" + dry_run gh -R "${GH_RELEASE}" release upload "${VERSION}" "./artifacts/${outfile}.yaml" fi fi done + dry_run rm -r "./artifacts/kustomize" } args "$@" diff --git a/kustomize/stage/fast/kustomization.yaml b/kustomize/stage/fast/kustomization.yaml new file mode 100644 index 000000000..499458f95 --- /dev/null +++ b/kustomize/stage/fast/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../node/fast +- ../pod/fast diff --git a/kustomize/stage/node/fast/README.md b/kustomize/stage/node/fast/README.md new file mode 100644 index 000000000..992c5aee0 --- /dev/null +++ b/kustomize/stage/node/fast/README.md @@ -0,0 +1,7 @@ +# Node Fast Stage + +This Stage only initializes the node. + +The `node-initialize` Stage is applied to nodes that do not have any conditions set in their `status.conditions` field. +When applied, this Stage sets the `status.conditions` field for the node, as well as the `status.addresses`, `status.allocatable`, +and `status.capacity` fields. diff --git a/kustomize/stage/node/fast/kustomization.yaml b/kustomize/stage/node/fast/kustomization.yaml new file mode 100644 index 000000000..7a7857ae8 --- /dev/null +++ b/kustomize/stage/node/fast/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- node-initialize.yaml diff --git a/kustomize/stage/node/fast/node-initialize.yaml b/kustomize/stage/node/fast/node-initialize.yaml new file mode 100644 index 000000000..127cd1036 --- /dev/null +++ b/kustomize/stage/node/fast/node-initialize.yaml @@ -0,0 +1,78 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: node-initialize +spec: + resourceRef: + apiGroup: v1 + kind: Node + selector: + matchExpressions: + - key: '.status.conditions.[] | select( .type == "Ready" ) | .status' + operator: 'NotIn' + values: + - 'True' + next: + statusTemplate: | + {{ $now := Now }} + {{ $lastTransitionTime := or .creationTimestamp $now }} + conditions: + {{ range NodeConditions }} + - lastHeartbeatTime: {{ $now | Quote }} + lastTransitionTime: {{ $lastTransitionTime | Quote }} + message: {{ .message | Quote }} + reason: {{ .reason | Quote }} + status: {{ .status | Quote }} + type: {{ .type | Quote}} + {{ end }} + + addresses: + {{ with .status.addresses }} + {{ YAML . 1 }} + {{ else }} + {{ with NodeIP }} + - address: {{ . | Quote }} + type: InternalIP + {{ end }} + {{ with NodeName }} + - address: {{ . | Quote }} + type: Hostname + {{ end }} + {{ end }} + + {{ with NodePort }} + daemonEndpoints: + kubeletEndpoint: + Port: {{ . }} + {{ end }} + + allocatable: + {{ with .status.allocatable }} + {{ YAML . 1 }} + {{ else }} + cpu: 1k + memory: 1Ti + pods: 1M + {{ end }} + capacity: + {{ with .status.capacity }} + {{ YAML . 1 }} + {{ else }} + cpu: 1k + memory: 1Ti + pods: 1M + {{ end }} + {{ with .status.nodeInfo }} + nodeInfo: + architecture: {{ with .architecture }} {{ . }} {{ else }} "amd64" {{ end }} + bootID: {{ with .bootID }} {{ . }} {{ else }} "" {{ end }} + containerRuntimeVersion: {{ with .containerRuntimeVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + kernelVersion: {{ with .kernelVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + kubeProxyVersion: {{ with .kubeProxyVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + kubeletVersion: {{ with .kubeletVersion }} {{ . }} {{ else }} "kwok-{{ Version }}" {{ end }} + machineID: {{ with .machineID }} {{ . }} {{ else }} "" {{ end }} + operatingSystem: {{ with .operatingSystem }} {{ . }} {{ else }} "linux" {{ end }} + osImage: {{ with .osImage }} {{ . }} {{ else }} "" {{ end }} + systemUUID: {{ with .systemUUID }} {{ . }} {{ else }} "" {{ end }} + {{ end }} + phase: Running diff --git a/kustomize/stage/node/heartbeat/README.md b/kustomize/stage/node/heartbeat/README.md new file mode 100644 index 000000000..69d56794f --- /dev/null +++ b/kustomize/stage/node/heartbeat/README.md @@ -0,0 +1,6 @@ +# Node Heartbeat Stage (DEPRECATED) + +This Stage configures the node heartbeat, only for prior to v0.3. + +The `node-heartbeat` Stage is applied to nodes that have the `Ready` condition set to `True` in their `status.conditions` field. +When applied, this Stage maintains the `status.conditions` field for the node. diff --git a/kustomize/stage/node/heartbeat/kustomization.yaml b/kustomize/stage/node/heartbeat/kustomization.yaml new file mode 100644 index 000000000..ffa886c18 --- /dev/null +++ b/kustomize/stage/node/heartbeat/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- node-heartbeat.yaml diff --git a/kustomize/stage/node/heartbeat/node-heartbeat.yaml b/kustomize/stage/node/heartbeat/node-heartbeat.yaml new file mode 100644 index 000000000..e67f224c2 --- /dev/null +++ b/kustomize/stage/node/heartbeat/node-heartbeat.yaml @@ -0,0 +1,35 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: node-heartbeat +spec: + resourceRef: + apiGroup: v1 + kind: Node + selector: + matchExpressions: + - key: '.status.phase' + operator: 'In' + values: + - 'Running' + - key: '.status.conditions.[] | select( .type == "Ready" ) | .status' + operator: 'In' + values: + - 'True' + delay: + durationMilliseconds: 20000 + jitterDurationMilliseconds: 25000 + next: + statusTemplate: | + {{ $now := Now }} + {{ $lastTransitionTime := or .creationTimestamp $now }} + conditions: + {{ range NodeConditions }} + - lastHeartbeatTime: {{ $now | Quote }} + lastTransitionTime: {{ $lastTransitionTime | Quote }} + message: {{ .message | Quote }} + reason: {{ .reason | Quote }} + status: {{ .status | Quote }} + type: {{ .type | Quote }} + {{ end }} + immediateNextStage: true diff --git a/kustomize/stage/pod/fast/README.md b/kustomize/stage/pod/fast/README.md new file mode 100644 index 000000000..b884f1947 --- /dev/null +++ b/kustomize/stage/pod/fast/README.md @@ -0,0 +1,15 @@ +# Pod Fast Stage + +These Stages make the pod ready, completed or deleted. + +The `pod-ready` Stage is applied to pods that do not have a `status.podIP` set and do not have a `metadata.deletionTimestamp` set. +When applied, this Stage sets the `status.conditions`, `status.containerStatuses`, and `status.initContainerStatuses` fields for the pod, +as well as the `status.hostIP` and `status.podIP` fields. It will also set the phase and startTime fields, indicating that the pod is running and has been started. + +The `pod-complete` Stage is applied to pods that are running, do not have a `metadata.deletionTimestamp` set, +and are owned by a Job. When applied, this Stage updates the `status.containerStatuses` field for the pod, +setting the ready and started fields to true and the `state.terminated` field to indicate that the pod has completed. +It also sets the phase field to Succeeded, indicating that the pod has completed successfully. + +The `pod-delete` Stage is applied to pods that have a `metadata.deletionTimestamp` set. +When applied, this Stage empties the `metadata.finalizers` field for the pod, allowing it to be deleted, and then delete the pod. diff --git a/kustomize/stage/pod/fast/kustomization.yaml b/kustomize/stage/pod/fast/kustomization.yaml new file mode 100644 index 000000000..2f8b975db --- /dev/null +++ b/kustomize/stage/pod/fast/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- pod-ready.yaml +- pod-complete.yaml +- pod-delete.yaml diff --git a/kustomize/stage/pod/fast/pod-complete.yaml b/kustomize/stage/pod/fast/pod-complete.yaml new file mode 100644 index 000000000..146a9489e --- /dev/null +++ b/kustomize/stage/pod/fast/pod-complete.yaml @@ -0,0 +1,40 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-complete +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.phase' + operator: 'In' + values: + - 'Running' + - key: '.metadata.ownerReferences.[].kind' + operator: 'In' + values: + - 'Job' + next: + statusTemplate: | + {{ $now := Now }} + {{ $root := . }} + containerStatuses: + {{ range $index, $item := .spec.containers }} + {{ $origin := index $root.status.containerStatuses $index }} + - image: {{ $item.image | Quote }} + name: {{ $item.name | Quote }} + ready: false + restartCount: 0 + started: false + state: + terminated: + exitCode: 0 + finishedAt: {{ $now | Quote }} + reason: Completed + startedAt: {{ $now | Quote }} + {{ end }} + phase: Succeeded diff --git a/kustomize/stage/pod/fast/pod-delete.yaml b/kustomize/stage/pod/fast/pod-delete.yaml new file mode 100644 index 000000000..69aa65450 --- /dev/null +++ b/kustomize/stage/pod/fast/pod-delete.yaml @@ -0,0 +1,16 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-delete +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'Exists' + next: + finalizers: + empty: true + delete: true diff --git a/kustomize/stage/pod/fast/pod-ready.yaml b/kustomize/stage/pod/fast/pod-ready.yaml new file mode 100644 index 000000000..253f00cb2 --- /dev/null +++ b/kustomize/stage/pod/fast/pod-ready.yaml @@ -0,0 +1,63 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-ready +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.podIP' + operator: 'DoesNotExist' + next: + statusTemplate: | + {{ $now := Now }} + + conditions: + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: Initialized + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: Ready + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: ContainersReady + {{ range .spec.readinessGates }} + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: {{ .conditionType | Quote }} + {{ end }} + + containerStatuses: + {{ range .spec.containers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: true + restartCount: 0 + state: + running: + startedAt: {{ $now | Quote }} + {{ end }} + + initContainerStatuses: + {{ range .spec.initContainers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: true + restartCount: 0 + state: + terminated: + exitCode: 0 + finishedAt: {{ $now | Quote }} + reason: Completed + startedAt: {{ $now | Quote }} + {{ end }} + + hostIP: {{ NodeIPWith .spec.nodeName | Quote }} + podIP: {{ PodIPWith .spec.nodeName ( or .spec.hostNetwork false ) ( or .metadata.uid "" ) ( or .metadata.name "" ) ( or .metadata.namespace "" ) | Quote }} + phase: Running + startTime: {{ $now | Quote }} diff --git a/kustomize/stage/pod/general/README.md b/kustomize/stage/pod/general/README.md new file mode 100644 index 000000000..d290e3c25 --- /dev/null +++ b/kustomize/stage/pod/general/README.md @@ -0,0 +1,4 @@ +# Pod General Stage (WIP) + +These Stages simulate real Pod behavior as closely as possible in the future, +which is not perfect at the moment, so the refinement of this configuration is still a *Work In Progress*. diff --git a/kustomize/stage/pod/general/kustomization.yaml b/kustomize/stage/pod/general/kustomization.yaml new file mode 100644 index 000000000..57eb6bd1e --- /dev/null +++ b/kustomize/stage/pod/general/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- pod-create.yaml +- pod-init-container-running.yaml +- pod-init-container-completed.yaml +- pod-ready.yaml +- pod-complete.yaml +- pod-remove-finalizer.yaml +- pod-delete.yaml diff --git a/kustomize/stage/pod/general/pod-complete.yaml b/kustomize/stage/pod/general/pod-complete.yaml new file mode 100644 index 000000000..89eff0684 --- /dev/null +++ b/kustomize/stage/pod/general/pod-complete.yaml @@ -0,0 +1,49 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-complete +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.phase' + operator: 'In' + values: + - 'Running' + - key: '.status.conditions.[] | select( .type == "Ready" ) | .status' + operator: 'In' + values: + - 'True' + - key: '.metadata.ownerReferences.[].kind' + operator: 'In' + values: + - 'Job' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationMilliseconds: 5000 + next: + delete: false + statusTemplate: | + {{ $now := Now }} + {{ $root := . }} + containerStatuses: + {{ range $index, $item := .spec.containers }} + {{ $origin := index $root.status.containerStatuses $index }} + - image: {{ $item.image | Quote }} + name: {{ $item.name | Quote }} + ready: true + restartCount: 0 + started: false + state: + terminated: + exitCode: 0 + finishedAt: {{ $now | Quote }} + reason: Completed + startedAt: {{ $now | Quote }} + {{ end }} + phase: Succeeded diff --git a/kustomize/stage/pod/general/pod-create.yaml b/kustomize/stage/pod/general/pod-create.yaml new file mode 100644 index 000000000..4d0f5372e --- /dev/null +++ b/kustomize/stage/pod/general/pod-create.yaml @@ -0,0 +1,101 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-create +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.podIP' + operator: 'DoesNotExist' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationMilliseconds: 5000 + next: + event: + type: Normal + reason: Created + message: Created container + finalizers: + add: + - value: 'kwok.x-k8s.io/fake' + statusTemplate: | + {{ $now := Now }} + + conditions: + {{ if .spec.initContainers }} + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + message: 'containers with incomplete status: [{{ range .spec.initContainers }} {{ .name }} {{ end }}]' + reason: ContainersNotInitialized + status: "False" + type: Initialized + {{ else }} + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + status: "True" + type: Initialized + {{ end }} + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + message: 'containers with unready status: [{{ range .spec.containers }} {{ .name }} {{ end }}]' + reason: ContainersNotReady + status: "False" + type: Ready + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + message: 'containers with unready status: [{{ range .spec.containers }} {{ .name }} {{ end }}]' + reason: ContainersNotReady + status: "False" + type: ContainersReady + {{ range .spec.readinessGates }} + - lastTransitionTime: {{ $now | Quote }} + status: "True" + type: {{ .conditionType | Quote }} + {{ end }} + + {{ if .spec.initContainers }} + initContainerStatuses: + {{ range .spec.initContainers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: false + restartCount: 0 + started: false + state: + waiting: + reason: PodInitializing + {{ end }} + containerStatuses: + {{ range .spec.containers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: false + restartCount: 0 + started: false + state: + waiting: + reason: PodInitializing + {{ end }} + {{ else }} + containerStatuses: + {{ range .spec.containers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: false + restartCount: 0 + started: false + state: + waiting: + reason: ContainerCreating + {{ end }} + {{ end }} + + hostIP: {{ NodeIPWith .spec.nodeName | Quote }} + podIP: {{ PodIPWith .spec.nodeName ( or .spec.hostNetwork false ) ( or .metadata.uid "" ) ( or .metadata.name "" ) ( or .metadata.namespace "" ) | Quote }} + phase: Pending diff --git a/kustomize/stage/pod/general/pod-delete.yaml b/kustomize/stage/pod/general/pod-delete.yaml new file mode 100644 index 000000000..f477a816e --- /dev/null +++ b/kustomize/stage/pod/general/pod-delete.yaml @@ -0,0 +1,21 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-delete +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'Exists' + - key: '.metadata.finalizers' + operator: 'DoesNotExist' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationFrom: + expressionFrom: '.metadata.deletionTimestamp' + next: + delete: true diff --git a/kustomize/stage/pod/general/pod-init-container-completed.yaml b/kustomize/stage/pod/general/pod-init-container-completed.yaml new file mode 100644 index 000000000..590f0a30f --- /dev/null +++ b/kustomize/stage/pod/general/pod-init-container-completed.yaml @@ -0,0 +1,61 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-init-container-completed +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.phase' + operator: 'In' + values: + - 'Pending' + - key: '.status.conditions.[] | select( .type == "Initialized" ) | .status' + operator: 'NotIn' + values: + - 'True' + - key: '.status.initContainerStatuses.[].state.running.startedAt' + operator: 'Exists' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationMilliseconds: 5000 + next: + statusTemplate: | + {{ $now := Now }} + {{ $root := . }} + conditions: + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + status: "True" + type: Initialized + initContainerStatuses: + {{ range $index, $item := .spec.initContainers }} + {{ $origin := index $root.status.initContainerStatuses $index }} + - image: {{ $item.image | Quote }} + name: {{ $item.name | Quote }} + ready: true + restartCount: 0 + started: false + state: + terminated: + exitCode: 0 + finishedAt: {{ $now | Quote }} + reason: Completed + startedAt: {{ $now | Quote }} + {{ end }} + containerStatuses: + {{ range .spec.containers }} + - image: {{ .image | Quote }} + name: {{ .name | Quote }} + ready: false + restartCount: 0 + started: false + state: + waiting: + reason: ContainerCreating + {{ end }} diff --git a/kustomize/stage/pod/general/pod-init-container-running.yaml b/kustomize/stage/pod/general/pod-init-container-running.yaml new file mode 100644 index 000000000..84a7d2c0c --- /dev/null +++ b/kustomize/stage/pod/general/pod-init-container-running.yaml @@ -0,0 +1,42 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-init-container-running +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.phase' + operator: 'In' + values: + - 'Pending' + - key: '.status.conditions.[] | select( .type == "Initialized" ) | .status' + operator: 'NotIn' + values: + - 'True' + - key: '.status.initContainerStatuses.[].state.waiting.reason' + operator: 'Exists' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationMilliseconds: 5000 + next: + statusTemplate: | + {{ $now := Now }} + {{ $root := . }} + initContainerStatuses: + {{ range $index, $item := .spec.initContainers }} + {{ $origin := index $root.status.initContainerStatuses $index }} + - image: {{ $item.image | Quote }} + name: {{ $item.name | Quote }} + ready: true + restartCount: 0 + started: true + state: + running: + startedAt: {{ $now | Quote }} + {{ end }} diff --git a/kustomize/stage/pod/general/pod-ready.yaml b/kustomize/stage/pod/general/pod-ready.yaml new file mode 100644 index 000000000..b4e114852 --- /dev/null +++ b/kustomize/stage/pod/general/pod-ready.yaml @@ -0,0 +1,59 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-ready +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'DoesNotExist' + - key: '.status.phase' + operator: 'In' + values: + - 'Pending' + - key: '.status.conditions.[] | select( .type == "Initialized" ) | .status' + operator: 'In' + values: + - 'True' + - key: '.status.conditions.[] | select( .type == "ContainersReady" ) | .status' + operator: 'NotIn' + values: + - 'True' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationMilliseconds: 5000 + next: + delete: false + statusTemplate: | + {{ $now := Now }} + {{ $root := . }} + conditions: + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + message: '' + reason: '' + status: "True" + type: Ready + - lastProbeTime: null + lastTransitionTime: {{ $now | Quote }} + message: '' + reason: '' + status: "True" + type: ContainersReady + containerStatuses: + {{ range $index, $item := .spec.containers }} + {{ $origin := index $root.status.containerStatuses $index }} + - image: {{ $item.image | Quote }} + name: {{ $item.name | Quote }} + ready: true + restartCount: 0 + started: true + state: + running: + startedAt: {{ $now | Quote }} + {{ end }} + phase: Running diff --git a/kustomize/stage/pod/general/pod-remove-finalizer.yaml b/kustomize/stage/pod/general/pod-remove-finalizer.yaml new file mode 100644 index 000000000..49f7e5cee --- /dev/null +++ b/kustomize/stage/pod/general/pod-remove-finalizer.yaml @@ -0,0 +1,28 @@ +apiVersion: kwok.x-k8s.io/v1alpha1 +kind: Stage +metadata: + name: pod-remove-finalizer +spec: + resourceRef: + apiGroup: v1 + kind: Pod + selector: + matchExpressions: + - key: '.metadata.deletionTimestamp' + operator: 'Exists' + - key: '.metadata.finalizers.[]' + operator: 'In' + values: + - 'kwok.x-k8s.io/fake' + weight: 1 + delay: + durationMilliseconds: 1000 + jitterDurationMilliseconds: 5000 + next: + finalizers: + remove: + - value: 'kwok.x-k8s.io/fake' + event: + type: Normal + reason: Killing + message: Stopping container diff --git a/site/content/en/docs/user/stages-configuration.md b/site/content/en/docs/user/stages-configuration.md index 2f128aa76..69be4f818 100644 --- a/site/content/en/docs/user/stages-configuration.md +++ b/site/content/en/docs/user/stages-configuration.md @@ -82,32 +82,6 @@ This example shows how to configure the simplest and fastest stages of Node reso [Default Node Stages] -``` goat { height=300 width=400 } - o - | - | Node be created that managed by kwok - v - .---------------. -| node-initialize | - '-------+-------' - | - | Update heartbeat - v - .---------------. -| node-heartbeat | - '-------+-------' - | ^ - | | Update heartbeat - '--' -``` - -The `node-initialize` Stage is applied to nodes that do not have any conditions set in their `status.conditions` field. -When applied, this Stage sets the `status.conditions` field for the node, as well as the `status.addresses`, `status.allocatable`, -and `status.capacity` fields. - -The `node-heartbeat` Stage is applied to nodes that have the `Ready` condition set to `True` in their `status.conditions` field. -When applied, this Stage maintains the `status.conditions` field for the node. - ### Pod Stages This example shows how to configure the simplest and fastest stages of Pod resource, which is also the default Pod stages for `kwok`. @@ -148,33 +122,18 @@ This example shows how to configure the simplest and fastest stages of Pod resou o ``` -The `pod-ready` Stage is applied to pods that do not have a `status.podIP` set and do not have a `metadata.deletionTimestamp` set. -When applied, this Stage sets the `status.conditions`, `status.containerStatuses`, and `status.initContainerStatuses` fields for the pod, -as well as the `status.hostIP` and `status.podIP` fields. It will also set the phase and startTime fields, indicating that the pod is running and has been started. - -The `pod-complete` Stage is applied to pods that are running, do not have a `metadata.deletionTimestamp` set, -and are owned by a Job. When applied, this Stage updates the `status.containerStatuses` field for the pod, -setting the ready and started fields to true and the `state.terminated` field to indicate that the pod has completed. -It also sets the phase field to Succeeded, indicating that the pod has completed successfully. - -The `pod-delete` Stage is applied to pods that have a `metadata.deletionTimestamp` set. -When applied, this Stage empties the `metadata.finalizers` field for the pod, allowing it to be deleted, and then delete the pod. - ### Pod Stages that simulate real behavior as closely as possible [General Pod Stages] -These Stages simulate real Pod behavior as closely as possible in the future, -which is not perfect at the moment, so the refinement of this configuration is still a **Work In Progress**. - [configuration]: {{< relref "/docs/user/configuration" >}} [Go Implementation]: https://github.com/itchyny/gojq [JQ Expressions]: https://stedolan.github.io/jq/manual/#Basicfilters -[Default Node Stages]: https://github.com/kubernetes-sigs/kwok/blob/main/stages/node-fast.yaml -[Default Pod Stages]: https://github.com/kubernetes-sigs/kwok/blob/main/stages/pod-fast.yaml -[General Pod Stages]: https://github.com/kubernetes-sigs/kwok/blob/main/stages/pod-general.yaml +[Default Node Stages]: https://github.com/kubernetes-sigs/kwok/tree/release/manifests/kustomize/stage/node/fast +[Default Pod Stages]: https://github.com/kubernetes-sigs/kwok/tree/release/manifests/kustomize/stage/pod/fast +[General Pod Stages]: https://github.com/kubernetes-sigs/kwok/tree/release/manifests/kustomize/stage/pod/general [Stage API]: {{< relref "/docs/generated/apis" >}}#kwok.x-k8s.io/v1alpha1.Stage diff --git a/test/release/build.test.sh b/test/release/build.test.sh index c7b5601c3..314bd4ddd 100755 --- a/test/release/build.test.sh +++ b/test/release/build.test.sh @@ -424,8 +424,8 @@ function want_manifests() { echo "cp -r ./kustomize ./artifacts/" echo "cd ./artifacts/kustomize/kwok" echo "kustomize edit set image registry.k8s.io/kwok/kwok=${IMAGE_PREFIX}/kwok:${VERSION}" - echo "kustomize build . -o ../../kwok.yaml" echo "cd -" + echo "kustomize build ./artifacts/kustomize/kwok -o ./artifacts/kwok.yaml" echo "rm -r ./artifacts/kustomize" } @@ -434,10 +434,10 @@ function manifests_with_push_ghrelease() { echo "cp -r ./kustomize ./artifacts/" echo "cd ./artifacts/kustomize/kwok" echo "kustomize edit set image registry.k8s.io/kwok/kwok=${IMAGE_PREFIX}/kwok:${VERSION}" - echo "kustomize build . -o ../../kwok.yaml" echo "cd -" - echo "rm -r ./artifacts/kustomize" + echo "kustomize build ./artifacts/kustomize/kwok -o ./artifacts/kwok.yaml" echo "gh -R ghrelease release upload ${VERSION} ./artifacts/kwok.yaml" + echo "rm -r ./artifacts/kustomize" } function manifests_with_push_bucket() { @@ -445,10 +445,10 @@ function manifests_with_push_bucket() { echo "cp -r ./kustomize ./artifacts/" echo "cd ./artifacts/kustomize/kwok" echo "kustomize edit set image registry.k8s.io/kwok/kwok=${IMAGE_PREFIX}/kwok:${VERSION}" - echo "kustomize build . -o ../../kwok.yaml" echo "cd -" - echo "rm -r ./artifacts/kustomize" + echo "kustomize build ./artifacts/kustomize/kwok -o ./artifacts/kwok.yaml" echo "gsutil cp -P ./artifacts/kwok.yaml bucket/releases/${PREFIX}-${VERSION}/manifests/kwok.yaml" + echo "rm -r ./artifacts/kustomize" } function main() { @@ -513,9 +513,9 @@ function main() { make --no-print-directory -C "${ROOT_DIR}" BUILDER=podman PUSH=true cross-cluster-image | diff -u <(want_cross_cluster_image_podman_with_push) - || failed+=("cross-cluster-image-podman-with-push") make --no-print-directory -C "${ROOT_DIR}" BUILDER=podman PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cross-cluster-image | diff -u <(want_cross_cluster_image_podman_with_push_staging) - || failed+=("cross-cluster-image-podman-with-push-staging") - make --no-print-directory -C "${ROOT_DIR}" IMAGE_PREFIX=${IMAGE_PREFIX} manifests | diff -u <(want_manifests) - || failed+=("manifests") - make --no-print-directory -C "${ROOT_DIR}" PUSH=true GH_RELEASE=ghrelease IMAGE_PREFIX=${IMAGE_PREFIX} manifests | diff -u <(manifests_with_push_ghrelease) - || failed+=("manifests-with-push-ghrelease") - make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} IMAGE_PREFIX=${IMAGE_PREFIX} manifests | diff -u <(manifests_with_push_bucket) - || failed+=("manifests-with-push-bucket") + make --no-print-directory -C "${ROOT_DIR}" MANIFESTS=kwok IMAGE_PREFIX=${IMAGE_PREFIX} manifests | diff -u <(want_manifests) - || failed+=("manifests") + make --no-print-directory -C "${ROOT_DIR}" PUSH=true MANIFESTS=kwok GH_RELEASE=ghrelease IMAGE_PREFIX=${IMAGE_PREFIX} manifests | diff -u <(manifests_with_push_ghrelease) - || failed+=("manifests-with-push-ghrelease") + make --no-print-directory -C "${ROOT_DIR}" PUSH=true MANIFESTS=kwok BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} IMAGE_PREFIX=${IMAGE_PREFIX} manifests | diff -u <(manifests_with_push_bucket) - || failed+=("manifests-with-push-bucket") if [[ "${#failed[@]}" -ne 0 ]]; then echo "Error: Some tests failed"