Generate AWS Makefile via this tool

provider-ci/internal/pkg/generate.go

@@ -91,12 +91,6 @@ func GeneratePackage(opts GenerateOpts) error {
 		if d.IsDir() {
 			return nil
 		}
-		// TODO: Back this option out after we convert pulumi-aws to use a standard Makefile.
-		//
-		// This is tracked in https://github.com/pulumi/pulumi-aws/issues/2694
-		if v, ok := config["XskipMakefile"]; ok && v.(bool) && strings.HasSuffix(path, "Makefile") {
-			return nil
-		}
 		inPath := path
 		outPath, err := filepath.Rel(templateDir, path)
 		if err != nil {

provider-ci/internal/pkg/templates/bridged-provider/Makefile

@@ -130,6 +130,17 @@ ifneq ("$(wildcard upstream)","")
 	@$(SHELL) ./scripts/upstream.sh "$@" apply
 endif
 
+#{{- if .Config.XrunUpstreamTools }}#
+
+	# Ensure tool is installed
+	cd upstream-tools && yarn install --frozen-lockfile
+	# Apply all automated changes
+	cd upstream-tools && yarn --silent run apply
+	# Check for any pending replacements
+	cd upstream-tools && yarn --silent run check
+
+#{{- end }}#
+
 upstream.finalize:
 	@$(SHELL) ./scripts/upstream.sh "$@" end_rebase
 

provider-ci/providers/aws/config.yaml

@@ -10,7 +10,9 @@ env:
   AWS_REGION: "us-west-2"
 makeTemplate: bridged
 checkoutSubmodules: true
-XskipMakefile: true
+# TODO: remove XrunUpstreamTools flag after work to add docs replacement strategies to resources.go is completed
+# Tracked in in https://github.com/pulumi/pulumi-aws/issues/2757
+XrunUpstreamTools: true
 plugins:
   - name: archive
     version: "0.0.1"
@@ -45,6 +47,9 @@ actions:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
+
   preBuild:
     - name: Clear GitHub Actions Ubuntu runner disk space
       uses: jlumbroso/free-disk-space@main
@@ -71,13 +76,15 @@ extraTests:
       with:
         ref: ${{ env.PR_COMMIT_SHA }}
         submodules: true
+    - name: Make upstream
+      run: make upstream
     - uses: actions/setup-go@v4
       with:
-       go-version-file: 'provider/shim/go.mod'
-       cache-dependency-path: 'provider/shim/go.sum'
+       go-version-file: 'provider/go.mod'
+       cache-dependency-path: 'provider/go.sum'
     - name: go test
       run: |
-        cd provider/shim
+        cd upstream/shim
         go test -v -coverprofile="coverage.txt" .
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3

provider-ci/providers/aws/repo/.github/workflows/main.yml

@@ -430,6 +430,8 @@ jobs:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -458,13 +460,15 @@ jobs:
             with:
               ref: ${{ env.PR_COMMIT_SHA }}
               submodules: true
+          - name: Make upstream
+            run: make upstream
           - uses: actions/setup-go@v4
             with:
-              cache-dependency-path: provider/shim/go.sum
-              go-version-file: provider/shim/go.mod
+              cache-dependency-path: provider/go.sum
+              go-version-file: provider/go.mod
           - name: go test
             run: |
-              cd provider/shim
+              cd upstream/shim
               go test -v -coverprofile="coverage.txt" .
           - env:
               CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

provider-ci/providers/aws/repo/.github/workflows/master.yml

@@ -430,6 +430,8 @@ jobs:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -458,13 +460,15 @@ jobs:
             with:
               ref: ${{ env.PR_COMMIT_SHA }}
               submodules: true
+          - name: Make upstream
+            run: make upstream
           - uses: actions/setup-go@v4
             with:
-              cache-dependency-path: provider/shim/go.sum
-              go-version-file: provider/shim/go.mod
+              cache-dependency-path: provider/go.sum
+              go-version-file: provider/go.mod
           - name: go test
             run: |
-              cd provider/shim
+              cd upstream/shim
               go test -v -coverprofile="coverage.txt" .
           - env:
               CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

provider-ci/providers/aws/repo/.github/workflows/nightly-test.yml

@@ -303,6 +303,8 @@ jobs:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt

provider-ci/providers/aws/repo/.github/workflows/prerelease.yml

@@ -381,6 +381,8 @@ jobs:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -409,13 +411,15 @@ jobs:
             with:
               ref: ${{ env.PR_COMMIT_SHA }}
               submodules: true
+          - name: Make upstream
+            run: make upstream
           - uses: actions/setup-go@v4
             with:
-              cache-dependency-path: provider/shim/go.sum
-              go-version-file: provider/shim/go.mod
+              cache-dependency-path: provider/go.sum
+              go-version-file: provider/go.mod
           - name: go test
             run: |
-              cd provider/shim
+              cd upstream/shim
               go test -v -coverprofile="coverage.txt" .
           - env:
               CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

provider-ci/providers/aws/repo/.github/workflows/release.yml

@@ -400,6 +400,8 @@ jobs:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -428,13 +430,15 @@ jobs:
             with:
               ref: ${{ env.PR_COMMIT_SHA }}
               submodules: true
+          - name: Make upstream
+            run: make upstream
           - uses: actions/setup-go@v4
             with:
-              cache-dependency-path: provider/shim/go.sum
-              go-version-file: provider/shim/go.mod
+              cache-dependency-path: provider/go.sum
+              go-version-file: provider/go.mod
           - name: go test
             run: |
-              cd provider/shim
+              cd upstream/shim
               go test -v -coverprofile="coverage.txt" .
           - env:
               CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

provider-ci/providers/aws/repo/.github/workflows/run-acceptance-tests.yml

@@ -375,6 +375,8 @@ jobs:
         role-duration-seconds: 3600
         role-session-name: ${{ env.PROVIDER }}@githubActions
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
+    - name: Make upstream
+      run: make upstream
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -403,13 +405,15 @@ jobs:
             with:
               ref: ${{ env.PR_COMMIT_SHA }}
               submodules: true
+          - name: Make upstream
+            run: make upstream
           - uses: actions/setup-go@v4
             with:
-              cache-dependency-path: provider/shim/go.sum
-              go-version-file: provider/shim/go.mod
+              cache-dependency-path: provider/go.sum
+              go-version-file: provider/go.mod
           - name: go test
             run: |
-              cd provider/shim
+              cd upstream/shim
               go test -v -coverprofile="coverage.txt" .
           - env:
               CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

provider-ci/providers/aws/repo/Makefile

@@ -0,0 +1,138 @@
+# WARNING: This file is autogenerated - changes will be overwritten if not made via https://github.com/pulumi/ci-mgmt
+
+PACK := aws
+ORG := pulumi
+PROJECT := github.com/$(ORG)/pulumi-$(PACK)
+PROVIDER_PATH := provider/v6
+VERSION_PATH := $(PROVIDER_PATH)/pkg/version.Version
+TFGEN := pulumi-tfgen-$(PACK)
+PROVIDER := pulumi-resource-$(PACK)
+VERSION := $(shell pulumictl get version)
+JAVA_GEN := pulumi-java-gen
+JAVA_GEN_VERSION := v0.9.5
+TESTPARALLELISM := 10
+WORKING_DIR := $(shell pwd)
+PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p 2
+
+development: install_plugins provider build_sdks install_sdks
+
+build: install_plugins provider build_sdks install_sdks
+
+build_sdks: build_nodejs build_python build_go build_dotnet build_java
+
+install_go_sdk:
+
+install_java_sdk:
+
+install_python_sdk:
+
+install_sdks: install_dotnet_sdk install_python_sdk install_nodejs_sdk install_java_sdk
+
+only_build: build
+
+build_dotnet: DOTNET_VERSION := $(shell pulumictl get version --language dotnet)
+build_dotnet: upstream
+	pulumictl get version --language dotnet
+	$(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/
+	cd sdk/dotnet/ && \
+		printf "module fake_dotnet_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
+		echo "$(DOTNET_VERSION)" >version.txt && \
+		dotnet build /p:Version=$(DOTNET_VERSION)
+
+build_go: upstream
+	$(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/
+	cd sdk && go list "$$(grep -e "^module" go.mod | cut -d ' ' -f 2)/go/..." | xargs go build
+
+build_java: PACKAGE_VERSION := $(shell pulumictl get version --language generic)
+build_java: bin/pulumi-java-gen upstream
+	$(WORKING_DIR)/bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java  --build gradle-nexus
+	cd sdk/java/ && \
+		printf "module fake_java_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
+		gradle --console=plain build
+
+build_nodejs: VERSION := $(shell pulumictl get version --language javascript)
+build_nodejs: upstream
+	$(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/
+	cd sdk/nodejs/ && \
+		printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
+		yarn install && \
+		yarn run tsc && \
+		cp ../../README.md ../../LICENSE* package.json yarn.lock ./bin/ && \
+		sed -i.bak -e "s/\$${VERSION}/$(VERSION)/g" ./bin/package.json
+
+build_python: PYPI_VERSION := $(shell pulumictl get version --language python)
+build_python: upstream
+	$(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/
+	cd sdk/python/ && \
+		printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \
+		cp ../../README.md . && \
+		python3 setup.py clean --all 2>/dev/null && \
+		rm -rf ./bin/ ../python.bin/ && cp -R . ../python.bin && mv ../python.bin ./bin && \
+		sed -i.bak -e 's/^VERSION = .*/VERSION = "$(PYPI_VERSION)"/g' -e 's/^PLUGIN_VERSION = .*/PLUGIN_VERSION = "$(VERSION)"/g' ./bin/setup.py && \
+		rm ./bin/setup.py.bak && rm ./bin/go.mod && \
+		cd ./bin && python3 setup.py build sdist
+
+clean:
+	rm -rf sdk/{dotnet,nodejs,go,python}
+
+cleanup:
+	rm -r $(WORKING_DIR)/bin
+	rm -f provider/cmd/$(PROVIDER)/schema.go
+
+help:
+	@grep '^[^.#]\+:\s\+.*#' Makefile | \
+	sed "s/\(.\+\):\s*\(.*\) #\s*\(.*\)/`printf "\033[93m"`\1`printf "\033[0m"`	\3 [\2]/" | \
+	expand -t20
+
+install_dotnet_sdk:
+	mkdir -p $(WORKING_DIR)/nuget
+	find . -name '*.nupkg' -print -exec cp -p {} $(WORKING_DIR)/nuget \;
+
+install_nodejs_sdk:
+	yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin
+
+install_plugins:
+	[ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh
+	pulumi plugin install resource archive 0.0.1
+	pulumi plugin install resource tls 4.10.0
+	pulumi plugin install resource github 4.10.0
+	pulumi plugin install resource kubernetes 3.17.0
+	pulumi plugin install resource random 4.8.2
+	pulumi plugin install resource github 5.14.0
+
+lint_provider: provider
+	cd provider && golangci-lint run -c ../.golangci.yml
+
+provider: tfgen install_plugins
+	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION) -X github.com/hashicorp/terraform-provider-aws/version.ProviderVersion=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER))
+
+test:
+	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+
+tfgen: install_plugins upstream
+	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
+	$(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER)
+	(cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go)
+
+upstream:
+ifneq ("$(wildcard upstream)","")
+	@$(SHELL) ./scripts/upstream.sh "$@" apply
+endif
+
+	# Ensure tool is installed
+	cd upstream-tools && yarn install --frozen-lockfile
+	# Apply all automated changes
+	cd upstream-tools && yarn --silent run apply
+	# Check for any pending replacements
+	cd upstream-tools && yarn --silent run check
+
+upstream.finalize:
+	@$(SHELL) ./scripts/upstream.sh "$@" end_rebase
+
+upstream.rebase:
+	@$(SHELL) ./scripts/upstream.sh "$@" start_rebase
+
+bin/pulumi-java-gen:
+	$(shell pulumictl download-binary -n pulumi-language-java -v $(JAVA_GEN_VERSION) -r pulumi/pulumi-java)
+
+.PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase