From 1dbae739733a6862f49588e4c2fd53b24b8a4fb8 Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Sun, 4 Apr 2021 10:09:24 -0700 Subject: [PATCH] build: Add make nuke; speed-up make start (#5583) Signed-off-by: Alex Collins --- .gitignore | 3 +-- Makefile | 47 +++++++++++++++++++++++++++++--------------- Procfile | 7 ++++--- hack/free-port.sh | 8 ++++++++ hack/port-forward.sh | 5 +---- 5 files changed, 45 insertions(+), 25 deletions(-) create mode 100755 hack/free-port.sh diff --git a/.gitignore b/.gitignore index 5bb2fc438fdc..d0de06e0d855 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ git-ask-pass.sh /pkg/apiclient/event/event.swagger.json /pkg/apiclient/eventsource/eventsource.swagger.json /pkg/apiclient/info/info.swagger.json +/pkg/apiclient/pipeline/pipeline.swagger.json /pkg/apiclient/sensor/sensor.swagger.json /pkg/apiclient/workflow/workflow.swagger.json /pkg/apiclient/workflowarchive/workflow-archive.swagger.json @@ -37,5 +38,3 @@ git-ask-pass.sh /go-diagrams/ /.run/ pprof -pkg/apiclient/sensor/sensor.swagger.json -pkg/apiclient/eventsource/eventsource.swagger.json diff --git a/Makefile b/Makefile index e60ca105164d..0cea188825da 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,6 @@ else STATIC_FILES ?= $(shell [ $(DEV_BRANCH) = true ] && echo false || echo true) endif -$(info GIT_COMMIT=$(GIT_COMMIT) GIT_BRANCH=$(GIT_BRANCH) GIT_TAG=$(GIT_TAG) GIT_TREE_STATE=$(GIT_TREE_STATE) RELEASE_TAG=$(RELEASE_TAG) DEV_BRANCH=$(DEV_BRANCH) VERSION=$(VERSION) STATIC_FILES=$(STATIC_FILES)) - START_UI ?= $(shell [ "$(CI)" != "" ] && echo true || echo false) GOTEST ?= go test -v PROFILE ?= minimal @@ -57,7 +55,9 @@ endif # * `local` run the workflow–controller and argo-server as single replicas on the local machine (default) # * `kubernetes` run the workflow-controller and argo-server on the Kubernetes cluster RUN_MODE := local -K3D := $(shell if [[ "`which kubectl`" != '' ]] && [[ "`kubectl config current-context`" == "k3d-"* ]]; then echo true; else echo false; fi) +KUBECTX := $(shell [[ "`which kubectl`" != '' ]] && kubectl config current-context || echo none) +DOCKER_DESKTOP := $(shell [[ "$(KUBECTX)" == "docker-desktop" ]] && echo true || echo false) +K3D := $(shell [[ "$(KUBECTX)" == "k3d-"* ]] && echo true || echo false) LOG_LEVEL := debug UPPERIO_DB_DEBUG := 0 NAMESPACED := true @@ -70,6 +70,10 @@ endif ALWAYS_OFFLOAD_NODE_STATUS := false +$(info GIT_COMMIT=$(GIT_COMMIT) GIT_BRANCH=$(GIT_BRANCH) GIT_TAG=$(GIT_TAG) GIT_TREE_STATE=$(GIT_TREE_STATE) RELEASE_TAG=$(RELEASE_TAG) DEV_BRANCH=$(DEV_BRANCH) VERSION=$(VERSION)) +$(info KUBECTX=$(KUBECTX) DOCKER_DESKTOP=$(DOCKER_DESKTOP) K3D=$(K3D) DOCKER_PUSH=$(DOCKER_PUSH)) +$(info RUN_MODE=$(RUN_MODE) PROFILE=$(PROFILE) AUTH_MODE=$(AUTH_MODE) SECURE=$(SECURE) STATIC_FILES=$(STATIC_FILES) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) LOG_LEVEL=$(LOG_LEVEL) NAMESPACED=$(NAMESPACED)) + override LDFLAGS += \ -X github.com/argoproj/argo-workflows/v3.version=$(VERSION) \ -X github.com/argoproj/argo-workflows/v3.buildDate=${BUILD_DATE} \ @@ -103,7 +107,7 @@ PROTO_BINARIES := $(GOPATH)/bin/protoc-gen-gogo $(GOPATH)/bin/protoc-gen-gogofas # go_install,path define go_install - [ -e ./vendor ] || go mod vendor + go mod vendor go install -mod=vendor ./vendor/$(1) endef @@ -178,10 +182,10 @@ dist/argo-linux-s390x: GOARGS = GOOS=linux GOARCH=s390x dist/argo-%.gz: dist/argo-% gzip --force --keep dist/argo-$* -dist/argo-%: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS) +dist/argo-%: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS) go.sum CGO_ENABLED=0 $(GOARGS) go build -v -i -ldflags '${LDFLAGS} -extldflags -static' -o $@ ./cmd/argo -dist/argo: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS) +dist/argo: server/static/files.go argo-server.crt argo-server.key $(CLI_PKGS) go.sum ifeq ($(shell uname -s),Darwin) # if local, then build fast: use CGO and dynamic-linking go build -v -i -ldflags '${LDFLAGS}' -o $@ ./cmd/argo @@ -197,7 +201,7 @@ argo-server.key: .PHONY: cli-image cli-image: dist/argocli.image -dist/argocli.image: $(CLI_PKGS) argo-server.crt argo-server.key +dist/argocli.image: $(CLI_PKGS) go.sum argo-server.crt argo-server.key $(call docker_build,argocli) touch dist/argocli.image @@ -209,7 +213,7 @@ clis: dist/argo-linux-amd64.gz dist/argo-linux-arm64.gz dist/argo-linux-ppc64le. .PHONY: controller controller: dist/workflow-controller -dist/workflow-controller: $(CONTROLLER_PKGS) +dist/workflow-controller: $(CONTROLLER_PKGS) go.sum ifeq ($(shell uname -s),Darwin) # if local, then build fast: use CGO and dynamic-linking go build -v -i -ldflags '${LDFLAGS}' -o $@ ./cmd/workflow-controller @@ -220,13 +224,13 @@ endif .PHONY: controller-image controller-image: dist/controller.image -dist/controller.image: $(CONTROLLER_PKGS) Dockerfile +dist/controller.image: $(CONTROLLER_PKGS) go.sum Dockerfile $(call docker_build,workflow-controller) touch dist/controller.image # argoexec -dist/argoexec: $(ARGOEXEC_PKGS) +dist/argoexec: $(ARGOEXEC_PKGS) go.sum ifeq ($(shell uname -s),Darwin) CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS} -extldflags -static' -o $@ ./cmd/argoexec else @@ -243,7 +247,7 @@ dist/argoexec.image: dist/argoexec mv argoexec dist/ docker tag argoproj/argoexec-dev:latest argoproj/argoexec:latest else -dist/argoexec.image: $(ARGOEXEC_PKGS) +dist/argoexec.image: $(ARGOEXEC_PKGS) go.sum $(call docker_build,argoexec) endif touch dist/argoexec.image @@ -416,6 +420,18 @@ ifeq ($(RUN_MODE),kubernetes) kubectl -n $(KUBE_NAMESPACE) scale deploy/argo-server --replicas 1 endif +# nuke is like "clean" but attempts to return you to a state as if it had never been installed at all +# this only available in "safe" local development +ifeq (true,$(filter true, $(DOCKER_DESKTOP) $(K3D))) +nuke: + kubectl delete --ignore-not-found workflow,cronworkflow,clusterworkflowtemplate,workflowtemplate,workfloweventbinding --all || true + kubectl delete --ignore-not-found ns argo + kubectl delete --ignore-not-found -k manifests/base/crds/minimal + git clean -fxd + docker image rm argoproj/argoexec argoproj/argoexec-dev argoproj/argocli argoproj/workflow-controller || true + docker system prune -f +endif + .PHONY: argosay argosay: cd test/e2e/images/argosay/v2 && docker build . -t argoproj/argosay:v2 @@ -444,7 +460,7 @@ $(GOPATH)/bin/goreman: .PHONY: start ifeq ($(RUN_MODE),local) -start: install executor-image controller cli $(GOPATH)/bin/goreman +start: install controller cli $(GOPATH)/bin/goreman else start: install executor-image controller-image cli-image endif @@ -458,8 +474,7 @@ endif grep '127.0.0.1[[:blank:]]*mysql' /etc/hosts ./hack/port-forward.sh ifeq ($(RUN_MODE),local) - killall goreman node || true - env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start controller argo-server logs watch-pods $(shell [ $(START_UI) = false ]&& echo ui || echo) $(shell if [ -z $GREP_LOGS ]; then echo; else echo "| grep \"$(GREP_LOGS)\""; fi) + env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start $(shell if [ -z $GREP_LOGS ]; then echo; else echo "| grep \"$(GREP_LOGS)\""; fi) endif $(GOPATH)/bin/stern: @@ -478,7 +493,7 @@ watch-pods: -w .PHONY: wait -wait: +wait: executor-image # Wait for workflow controller until lsof -i :9090 > /dev/null ; do sleep 10s ; done # Wait for Argo Server @@ -575,7 +590,7 @@ docs/fields.md: api/openapi-spec/swagger.json $(shell find examples -type f) hac env ARGO_SECURE=false ARGO_INSECURE_SKIP_VERIFY=false ARGO_SERVER= ARGO_INSTANCEID= go run ./hack docgen # generates several other files -docs/cli/argo.md: $(CLI_PKGS) server/static/files.go hack/cli/main.go +docs/cli/argo.md: $(CLI_PKGS) go.sum server/static/files.go hack/cli/main.go go run ./hack/cli .PHONY: validate-examples diff --git a/Procfile b/Procfile index baa800fe8538..310571f6f247 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,6 @@ -controller: PNS_PRIVILEGED=true DEFAULT_REQUEUE_TIME=${DEFAULT_REQUEUE_TIME} LEADER_ELECTION_IDENTITY=local ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image ${IMAGE_NAMESPACE}/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --loglevel ${LOG_LEVEL} -argo-server: UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE --x-frame-options=SAMEORIGIN -ui: yarn --cwd ui install && yarn --cwd ui start +executor-image : make executor-image +controller: ./hack/free-port.sh 9090 && PNS_PRIVILEGED=true DEFAULT_REQUEUE_TIME=${DEFAULT_REQUEUE_TIME} LEADER_ELECTION_IDENTITY=local ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image ${IMAGE_NAMESPACE}/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --loglevel ${LOG_LEVEL} +argo-server: ./hack/free-port.sh 2746 && UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE --x-frame-options=SAMEORIGIN +ui: ./hack/free-port.sh 8080 && yarn --cwd ui install && yarn --cwd ui start logs: make logs watch-pods: make watch-pods diff --git a/hack/free-port.sh b/hack/free-port.sh new file mode 100755 index 000000000000..6cf3e2554c3b --- /dev/null +++ b/hack/free-port.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eux + +port=$1 + +lsof -i ":$port" | grep -v PID | awk '{print $2}' | xargs kill || true + + diff --git a/hack/port-forward.sh b/hack/port-forward.sh index beba9514df42..6890a7850a76 100755 --- a/hack/port-forward.sh +++ b/hack/port-forward.sh @@ -7,10 +7,7 @@ pf() { resource=$2 port=$3 dest_port=${4:-"$port"} - pid=$(lsof -i ":$port" | grep -v PID | awk '{print $2}' || true) - if [ "$pid" != "" ]; then - kill $pid - fi + ./hack/free-port.sh $port kubectl -n argo port-forward "$resource" "$port:$dest_port" > /dev/null & # wait until port forward is established until lsof -i ":$port" > /dev/null ; do sleep 1s ; done