Break provider-gcp up by service

Makefile

@@ -14,6 +14,8 @@ export TERRAFORM_DOCS_PATH ?= website/docs/r
 
 PLATFORMS ?= linux_amd64 linux_arm64
 
+export PROJECT_NAME := $(PROJECT_NAME)
+
 # -include will silently skip missing files, which allows us
 # to load those files with a target in the Makefile. If only
 # "include" was used, the make command would fail and refuse
@@ -43,19 +45,42 @@ export GOPRIVATE = github.com/upbound/*
 
 GO_REQUIRED_VERSION ?= 1.19
 GOLANGCILINT_VERSION ?= 1.50.0
-GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/provider $(GO_PROJECT)/cmd/generator
+# SUBPACKAGES ?= $(shell find cmd/provider -type d -maxdepth 1 -mindepth 1 | cut -d/ -f3)
+SUBPACKAGES ?= monolith
+GO_STATIC_PACKAGES ?= $(GO_PROJECT)/cmd/generator ${SUBPACKAGES:%=$(GO_PROJECT)/cmd/provider/%}
 GO_LDFLAGS += -X $(GO_PROJECT)/internal/version.Version=$(VERSION)
 GO_SUBDIRS += cmd internal apis
 GO111MODULE = on
+
+export SUBPACKAGES := $(SUBPACKAGES)
+
 -include build/makelib/golang.mk
 
+# ====================================================================================
+# Setup XPKG
+
+XPKG_REG_ORGS ?= xpkg.upbound.io/upbound
+# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
+# inferred.
+XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/upbound
+
+export XPKG_REG_ORGS := $(XPKG_REG_ORGS)
+export XPKG_REG_ORGS_NO_PROMOTE := $(XPKG_REG_ORGS_NO_PROMOTE)
+
+-include build/makelib/xpkg.mk
+
 # ====================================================================================
 # Setup Kubernetes tools
 
 KIND_VERSION = v0.15.0
+UPTEST_VERSION = v0.5.0
+# dependency for up
 UP_VERSION = v0.16.1
 UP_CHANNEL = stable
-UPTEST_VERSION = v0.5.0
+
+export UP_VERSION := $(UP_VERSION)
+export UP_CHANNEL := $(UP_CHANNEL)
+
 -include build/makelib/k8s_tools.mk
 
 # ====================================================================================
@@ -65,20 +90,6 @@ REGISTRY_ORGS ?= xpkg.upbound.io/upbound
 IMAGES = provider-gcp
 -include build/makelib/imagelight.mk
 
-# ====================================================================================
-# Setup XPKG
-
-XPKG_REG_ORGS ?= xpkg.upbound.io/upbound
-# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
-# inferred.
-XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/upbound
-XPKGS = provider-gcp
--include build/makelib/xpkg.mk
-
-# NOTE(hasheddan): we force image building to happen prior to xpkg build so that
-# we ensure image is present in daemon.
-xpkg.build.provider-gcp: do.build.images
-
 # ====================================================================================
 # Targets
 
@@ -169,9 +180,9 @@ uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
 uptest-local:
 	@$(WARN) "this target is deprecated, please use 'make uptest' instead"
 
-local-deploy: build controlplane.up local.xpkg.deploy.provider.$(PROJECT_NAME)
+local-deploy: build controlplane.up local.xpkg.deploy.provider.$(PROJECT_NAME)-monolith
 	@$(INFO) running locally built provider
-	@$(KUBECTL) wait provider.pkg $(PROJECT_NAME) --for condition=Healthy --timeout 5m
+	@$(KUBECTL) wait provider.pkg $(PROJECT_NAME)-monolith --for condition=Healthy --timeout 5m
 	@$(KUBECTL) -n upbound-system wait --for=condition=Available deployment --all --timeout=5m
 	@$(OK) running locally built provider
 

apis/generate.go

@@ -29,6 +29,8 @@ limitations under the License.
 //go:generate bash -c "find ../internal/controller -iname 'zz_*' -delete"
 //go:generate bash -c "find ../internal/controller -type d -empty -delete"
 //go:generate rm -rf ../examples-generated
+//go:generate bash -c "find ../cmd/provider -name 'zz_*' -type f -delete"
+//go:generate bash -c "find ../cmd/provider -type d -maxdepth 1 -mindepth 1 -empty -delete"
 
 // Scrape metadata from Terraform registry
 //go:generate go run github.com/upbound/upjet/cmd/scraper -n hashicorp/terraform-provider-google -r ../.work/terraform-provider-google/website/docs/r -o ../config/provider-metadata.yaml --prelude-xpath "//text()[contains(., \"subcategory\")]" --resource-prefix google

cluster/images/provider-gcp/Dockerfile

@@ -1,52 +1,61 @@
-FROM alpine:3.17.2
-RUN apk --no-cache add ca-certificates bash
+FROM alpine:3.16.2 AS builder
+RUN apk --no-cache add curl
 
 ARG TARGETOS
 ARG TARGETARCH
 
-ENV USER_ID=65532
-
 # Setup Terraform environment
 
 ## Provider-dependent configuration
-ARG CROSSPLANE_PROVIDER_VERSION
 ARG TERRAFORM_VERSION
-ARG TERRAFORM_PROVIDER_SOURCE
 ARG TERRAFORM_PROVIDER_VERSION
 ARG TERRAFORM_PROVIDER_DOWNLOAD_NAME
-ARG TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX
 ## End of - Provider-dependent configuration
 
-ENV PLUGIN_DIR /terraform/provider-mirror/registry.terraform.io/${TERRAFORM_PROVIDER_SOURCE}/${TERRAFORM_PROVIDER_VERSION}/${TARGETOS}_${TARGETARCH}
-ENV TF_CLI_CONFIG_FILE /terraform/.terraformrc
-ENV TF_FORK 0
+RUN curl -L -o /tmp/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip \
+  && unzip /tmp/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip -d /tmp/ \
+  && chmod +x /tmp/terraform \
+  && rm /tmp/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip \
+  && curl -L -o /tmp/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip https://releases.hashicorp.com/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}/${TERRAFORM_PROVIDER_VERSION}/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip \
+  && unzip /tmp/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip -d /tmp/native/ \
+  && chmod +x /tmp/native/* \
+  && rm /tmp/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip
+# End of - Setup Terraform environment
 
-RUN mkdir -p ${PLUGIN_DIR}
+FROM alpine:3.16.2 as base
 
-ADD https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip /tmp
-ADD https://releases.hashicorp.com/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}/${TERRAFORM_PROVIDER_VERSION}/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip /tmp
-ADD terraformrc.hcl ${TF_CLI_CONFIG_FILE}
+ARG CROSSPLANE_PROVIDER_VERSION
+ARG TARGETOS
+ARG TARGETARCH
+ARG APIGROUP
+ARG TERRAFORM_VERSION
+ARG TERRAFORM_PROVIDER_SOURCE
+ARG TERRAFORM_PROVIDER_VERSION
+ARG TERRAFORM_PROVIDER_DOWNLOAD_NAME
+ARG PLUGIN_DIR=/terraform/provider-mirror/registry.terraform.io/${TERRAFORM_PROVIDER_SOURCE}/${TERRAFORM_PROVIDER_VERSION}/${TARGETOS}_${TARGETARCH}
 
-RUN unzip /tmp/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip -d /usr/local/bin \
-  && chmod +x /usr/local/bin/terraform \
-  && rm /tmp/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip \
-  && unzip /tmp/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip -d ${PLUGIN_DIR} \
-  && chmod +x ${PLUGIN_DIR}/* \
-  && rm /tmp/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_${TERRAFORM_PROVIDER_VERSION}_${TARGETOS}_${TARGETARCH}.zip \
-  && chown -R ${USER_ID}:${USER_ID} /terraform
-# End of - Setup Terraform environment
+ENV USER_ID=65532
+ENV TF_CLI_CONFIG_FILE /terraform/.terraformrc
+ENV TF_FORK 0
+# set user agent
+ENV GOOGLE_TERRAFORM_USERAGENT_EXTENSION "upbound-provider-gcp/${CROSSPLANE_PROVIDER_VERSION}"
 
 # Provider controller needs these environment variable at runtime
 ENV TERRAFORM_VERSION ${TERRAFORM_VERSION}
 ENV TERRAFORM_PROVIDER_SOURCE ${TERRAFORM_PROVIDER_SOURCE}
 ENV TERRAFORM_PROVIDER_VERSION ${TERRAFORM_PROVIDER_VERSION}
-# ENV TERRAFORM_NATIVE_PROVIDER_PATH ${PLUGIN_DIR}/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_v${TERRAFORM_PROVIDER_VERSION}_x5
-ENV GOOGLE_TERRAFORM_USERAGENT_EXTENSION "upbound-provider-gcp/${CROSSPLANE_PROVIDER_VERSION}"
 ENV TERRAFORM_NATIVE_PROVIDER_PATH ${PLUGIN_DIR}/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_v${TERRAFORM_PROVIDER_VERSION}_x5
 
+RUN apk --no-cache add ca-certificates bash
+
 USER ${USER_ID}
 EXPOSE 8080
 
-ADD "bin/${TARGETOS}_${TARGETARCH}/provider" /usr/local/bin/provider
+ADD terraformrc.hcl ${TF_CLI_CONFIG_FILE}
+COPY --from=builder /tmp/terraform /usr/local/bin/
+COPY --from=builder /tmp/native/* ${PLUGIN_DIR}/
 
 ENTRYPOINT ["provider"]
+
+FROM base
+ADD "bin/${TARGETOS}_${TARGETARCH}/${APIGROUP}" /usr/local/bin/provider

cluster/images/provider-gcp/Makefile

@@ -8,23 +8,62 @@ include ../../../build/makelib/common.mk
 
 include ../../../build/makelib/imagelight.mk
 
+# ====================================================================================
+# XPKG
+
+-include ../../../build/makelib/xpkg.mk
+
+-include ../../../build/makelib/k8s_tools.mk
+
 # ====================================================================================
 # Targets
 
 img.build:
-	@$(INFO) docker build $(IMAGE)
-	@$(MAKE) BUILD_ARGS="--load" img.build.shared
-	@$(OK) docker build $(IMAGE)
+	@$(INFO) Subpackages to build: $(SUBPACKAGES)
+	@for g in $(SUBPACKAGES); do \
+		IMAGE=$$(echo $(IMAGE) | sed -r "s/(.*)-/\1-$${g}-/"); \
+		$(INFO) Building image $${IMAGE}; \
+		$(MAKE) BUILD_ARGS="--load" IMAGE=$${IMAGE} APIGROUP=$${g} XPKG_REG_ORGS=$(XPKG_REG_ORGS) img.build.shared; \
+		$(OK) docker build $${IMAGE}; \
+	done
+
+# make BRANCH_NAME=main SUBPACKAGES=cloudplatform publish
+img.publish: $(UP)
+	@$(INFO) Subpackages to publish: $(SUBPACKAGES)
+	@for g in $(SUBPACKAGES); do \
+		PACKAGE=$(XPKG_REG_ORGS)/$(PROJECT_NAME)-$${g}:$(VERSION); \
+		if [[ $${g} == "monolith" ]]; then PACKAGE=$(XPKG_REG_ORGS)/$(PROJECT_NAME):$(VERSION); fi; \
+		$(INFO) Pushing package $${PACKAGE}; \
+		$(UP) xpkg push $${PACKAGE} \
+				$(foreach p,$(XPKG_LINUX_PLATFORMS),--package $(XPKG_OUTPUT_DIR)/$(p)/$(PROJECT_NAME)-$${g}-$(VERSION).xpkg ) || $(FAIL); \
+		$(OK) Done processing $${PACKAGE}. Moving to the next package...; \
+	done || $(FAIL)
 
-img.publish:
-	@$(INFO) Skipping image publish for $(IMAGE)
-	@echo Publish is deferred to xpkg machinery
-	@$(OK) Image publish skipped for $(IMAGE)
+DEP_CONSTRAINT := >= 0.0.0
+ifeq (-,$(findstring -,$(VERSION)))
+    DEP_CONSTRAINT = >= 0.0.0-0
+endif
 
-img.build.shared:
+img.build.shared: $(UP)
+	@mkdir -p $(IMAGE_TEMP_DIR)/package/crds || $(FAIL)
+	@mkdir $(IMAGE_TEMP_DIR)/examples || $(FAIL)
+	@if [[ $(APIGROUP) == "config" ]]; then \
+		sed -e "s/{{ APIGROUP }}/$(APIGROUP)/g" $(ROOT_DIR)/package/crossplane-config.yaml > $(IMAGE_TEMP_DIR)/package/crossplane.yaml && \
+		cp $(ROOT_DIR)/package/crds/gcp.* $(IMAGE_TEMP_DIR)/package/crds/ && \
+		cp -r $(ROOT_DIR)/examples/gcp $(IMAGE_TEMP_DIR)/examples/; \
+	elif [[ $(APIGROUP) == "monolith" ]]; then \
+		cp $(ROOT_DIR)/package/crossplane-monolith.yaml $(IMAGE_TEMP_DIR)/package/crossplane.yaml && \
+		cp -r $(ROOT_DIR)/package/crds $(IMAGE_TEMP_DIR)/package/ && \
+		cp -r $(ROOT_DIR)/examples $(IMAGE_TEMP_DIR)/; \
+	else \
+		sed -e "s|{{ APIGROUP }}|$(APIGROUP)|g" $(ROOT_DIR)/package/crossplane.yaml | sed -e "s|{{ XPKG_REG_ORGS }}|$(XPKG_REG_ORGS)|g" | sed -e "s|{{ VERSION }}|$(VERSION)|g" | sed -e "s|{{ DEP_CONSTRAINT }}|$(DEP_CONSTRAINT)|g" > $(IMAGE_TEMP_DIR)/package/crossplane.yaml && \
+		cp $(ROOT_DIR)/package/crds/$(APIGROUP).* $(IMAGE_TEMP_DIR)/package/crds/ && \
+		cp -r $(ROOT_DIR)/examples/$(APIGROUP) $(IMAGE_TEMP_DIR)/examples/; \
+  	fi || $(FAIL)
 	@cp Dockerfile $(IMAGE_TEMP_DIR) || $(FAIL)
 	@cp terraformrc.hcl $(IMAGE_TEMP_DIR) || $(FAIL)
-	@cp -r $(OUTPUT_DIR)/bin/ $(IMAGE_TEMP_DIR)/bin || $(FAIL)
+	@mkdir -p $(IMAGE_TEMP_DIR)/bin/$(PLATFORM)/ || $(FAIL)
+	@cp -r $(OUTPUT_DIR)/bin/$(PLATFORM)/$(APIGROUP) $(IMAGE_TEMP_DIR)/bin/$(PLATFORM)/ || $(FAIL)
 	@docker buildx build $(BUILD_ARGS) \
 		--platform $(IMAGE_PLATFORMS) \
 		--build-arg TERRAFORM_VERSION=$(TERRAFORM_VERSION) \
@@ -33,8 +72,12 @@ img.build.shared:
 		--build-arg TERRAFORM_PROVIDER_DOWNLOAD_NAME=$(TERRAFORM_PROVIDER_DOWNLOAD_NAME) \
 		--build-arg TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX=$(TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX) \
 		--build-arg CROSSPLANE_PROVIDER_VERSION=$(VERSION) \
+		--build-arg APIGROUP=$(APIGROUP) \
 		-t $(IMAGE) \
 		$(IMAGE_TEMP_DIR) || $(FAIL)
+	@$(MAKE) UP=$(UP) XPKG_REG_ORGS=$(XPKG_REG_ORGS) XPKG_REG_ORGS_NO_PROMOTE=$(XPKG_REG_ORGS_NO_PROMOTE) XPKGS=$(PROJECT_NAME)-$(APIGROUP) XPKG_DIR=$(IMAGE_TEMP_DIR)/package XPKG_EXAMPLES_DIR=$(IMAGE_TEMP_DIR)/examples build.xpkg || $(FAIL)
+
+build.xpkg: do.build.xpkgs
 
 img.promote:
 	@$(INFO) Skipping image promotion from $(FROM_IMAGE) to $(TO_IMAGE)