diff --git a/.circleci/chocolatey.config b/.circleci/chocolatey.config index afa94a7dad..e3dc238e11 100644 --- a/.circleci/chocolatey.config +++ b/.circleci/chocolatey.config @@ -3,6 +3,5 @@ - diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ea2a6a974..af3e8c935e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,28 @@ version: '2.1' +parameters: + go_download_base_url: + type: string + default: 'https://storage.googleapis.com/golang/' + fips_go_download_base_url: + type: string + default: 'https://aka.ms/golang/release/latest/' + go_version: + type: string + # https://go.dev/doc/devel/release + default: '1.20.6' + aws_version: + type: string + # https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst + default: '2.4.12' + gh_version: + type: string + # https://github.com/cli/cli/releases + default: '2.7.0' + orbs: prodsec: snyk/prodsec-orb@1.0 snyk: snyk/snyk@1.7.0 - go: circleci/go@1.7.1 win: circleci/windows@5.0 aws-cli: circleci/aws-cli@2.0.3 gh: circleci/github-cli@2.1.0 @@ -22,11 +41,6 @@ executors: - image: bastiandoetsch209/cli-build-arm64:20230806-080507 working_directory: /mnt/ramdisk/snyk resource_class: arm.large - linux-amd64: - machine: - image: ubuntu-2204:2023.02.1 - working_directory: /mnt/ramdisk/snyk - resource_class: large linux-ubuntu-mantic-amd64: docker: - image: ubuntu:mantic @@ -77,8 +91,63 @@ executors: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions xcode: '14.3.1' resource_class: macos.m1.large.gen1 + win-amd64: + machine: + image: windows-server-2022-gui:2023.07.1 + resource_class: windows.large + shell: powershell commands: + install-go: + parameters: + go_os: + type: string + go_target_os: + type: string + go_arch: + type: string + base_url: + type: string + extraction_path: + type: string + cache_key_file: + type: string + default: go_cache_key.txt + steps: + - run: + name: Create Cache Key + command: | + echo << parameters.extraction_path >>-<< parameters.base_url >>-<< parameters.go_target_os >>-<< parameters.go_arch >>-<< pipeline.parameters.go_version >> > << parameters.cache_key_file >> + cat << parameters.cache_key_file >> + - restore_cache: + name: Restoring go binary cache + keys: + - go-binary-cache-{{ checksum "<< parameters.cache_key_file >>" }} + - run: + name: Download go binary + command: python ./scripts/download_go.py << pipeline.parameters.go_version >> --go_os=<< parameters.go_os >> --go_arch=<< parameters.go_arch >> --base_url=<< parameters.base_url >> --extraction_path=<< parameters.extraction_path >> + - save_cache: + name: Caching go binary + key: go-binary-cache-{{ checksum "<< parameters.cache_key_file >>" }} + paths: + - << parameters.extraction_path >>/go + - unless: + condition: + equal: ['windows', << parameters.go_os >>] + steps: + - run: + name: Add go binary to PATH + command: echo "export PATH=$(realpath << parameters.extraction_path >>/go/bin):\$PATH" >> "$BASH_ENV" + - when: + condition: + equal: ['windows', << parameters.go_os >>] + steps: + - run: + name: Add go binary to PATH + command: | + New-Item -Path $profile -ItemType File -Force + '$Env:Path = "<< parameters.extraction_path >>\go\bin;" + $Env:Path' >> $profile + install-deps-windows-full: steps: - restore_cache: @@ -158,20 +227,6 @@ commands: - attach_workspace: at: . -parameters: - go_version: - type: string - # https://go.dev/doc/devel/release - default: '1.20.6' - aws_version: - type: string - # https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst - default: '2.4.12' - gh_version: - type: string - # https://github.com/cli/cli/releases - default: '2.7.0' - #################################################################################################### # WORKFLOWS #################################################################################################### @@ -214,49 +269,94 @@ workflows: - build-artifact: name: build linux amd64 + go_target_os: linux go_os: linux go_arch: amd64 + go_download_base_url: << pipeline.parameters.go_download_base_url >> executor: docker-amd64 - artifact: snyk-linux requires: - prepare-build - build-artifact: name: build linux arm64 + go_target_os: linux go_os: linux go_arch: arm64 + go_download_base_url: << pipeline.parameters.go_download_base_url >> executor: docker-arm64 - artifact: snyk-linux-arm64 requires: - prepare-build - build-artifact: name: build alpine amd64 - go_os: alpine + go_target_os: alpine + go_os: linux go_arch: amd64 + go_download_base_url: << pipeline.parameters.go_download_base_url >> executor: docker-amd64 - artifact: snyk-alpine c_compiler: /usr/bin/musl-gcc requires: - prepare-build - build-artifact: name: build macOS amd64 + go_target_os: darwin go_os: darwin go_arch: amd64 + go_download_base_url: << pipeline.parameters.go_download_base_url >> executor: macos-amd64 - artifact: snyk-macos requires: - prepare-build - - build-windows-artifact: - context: snyk-windows-signing + - build-artifact: name: build windows amd64 + go_target_os: windows go_os: windows go_arch: amd64 + go_download_base_url: << pipeline.parameters.go_download_base_url >> + install_deps_extension: windows-full + install_path: 'C:\' + executor: win-amd64 + context: snyk-windows-signing requires: - prepare-build + # - build-artifact: + # name: build windows amd64 + # go_target_os: windows + # go_os: windows + # go_arch: amd64 + # go_download_base_url: << pipeline.parameters.fips_go_download_base_url >> + # make_target: build-fips + # install_deps_extension: windows-full + # install_path: 'C:\' + # executor: win-amd64 + # context: snyk-windows-signing + # requires: + # - prepare-build + # + # - build-artifact: + # name: fips build linux amd64 + # go_target_os: linux + # go_os: linux + # go_arch: amd64 + # go_download_base_url: << pipeline.parameters.fips_go_download_base_url >> + # make_target: build-fips + # executor: docker-amd64 + # requires: + # - prepare-build + # + # - build-artifact: + # name: fips build linux arm64 + # go_target_os: linux + # go_os: linux + # go_arch: arm64 + # go_download_base_url: << pipeline.parameters.fips_go_download_base_url >> + # make_target: build-fips + # executor: docker-arm64 + # requires: + # - prepare-build + - regression-tests: name: regression-tests context: nodejs-install @@ -515,7 +615,7 @@ jobs: command: | npm run lint pushd cliv2 - make lint configure + make lint popd - snyk/scan: fail-on-issues: true @@ -540,7 +640,7 @@ jobs: - run: name: Running Go unit tests working_directory: ./cliv2 - command: make configure whiteboxtest + command: make whiteboxtest - run: name: Running Tap tests command: @@ -563,36 +663,12 @@ jobs: - binary-releases/snyk-fix.tgz - binary-releases/snyk-protect.tgz - build-windows-artifact: - parameters: - go_os: - type: string - go_arch: - type: string - executor: - name: win/default - size: large - steps: - - prepare-workspace - - install-deps-windows-full - - run: - name: Build Windows - shell: powershell - command: make build GOOS=windows GOARCH=amd64 - environment: - CGO_ENABLED: 1 - - store_artifacts: - path: binary-releases - - persist_to_workspace: - root: . - paths: - - binary-releases/snyk-* - - binary-releases/version - build-artifact: parameters: go_os: type: string + go_target_os: + type: string go_arch: type: string c_compiler: @@ -600,23 +676,35 @@ jobs: default: '' executor: type: string - artifact: + go_download_base_url: + type: string + make_target: type: string + default: 'build' + install_deps_extension: + type: string + default: 'noop' + install_path: + type: string + default: '.' executor: << parameters.executor >> - environment: - HOSTTYPE: << parameters.go_arch >> steps: - prepare-workspace - - go/install: - version: << pipeline.parameters.go_version >> + - install-deps-<< parameters.install_deps_extension >> + - install-go: + go_os: << parameters.go_os >> + go_target_os: << parameters.go_target_os >> + go_arch: << parameters.go_arch >> + base_url: << parameters.go_download_base_url >> + extraction_path: << parameters.install_path >> - restore_cache: key: go-build-{{ arch }}-{{ checksum "cliv2/go.sum" }} - run: - name: Build << parameters.go_os >>/<< parameters.go_arch >> + name: Build << parameters.go_target_os >>/<< parameters.go_arch >> environment: CC: << parameters.c_compiler >> CGO_ENABLED: 1 - command: make build GOOS=<< parameters.go_os >> GOARCH=<< parameters.go_arch >> + command: make << parameters.make_target >> GOOS=<< parameters.go_target_os >> GOARCH=<< parameters.go_arch >> - save_cache: key: go-build-{{ arch }}-{{ checksum "cliv2/go.sum" }} paths: [/home/circleci/go/pkg/mod] diff --git a/Makefile b/Makefile index 30faff3467..eea9ea23e2 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,10 @@ BINARY_RELEASES_FOLDER_TS_CLI = binary-releases BINARY_OUTPUT_FOLDER = binary-releases SHASUM_CMD = shasum GOHOSTOS = $(shell go env GOHOSTOS) -PYTHON = python +export PYTHON = python PYTHON_VERSION = $(shell python3 --version) -ifdef (PYTHON_VERSION) +ifneq (, $(PYTHON_VERSION)) PYTHON = python3 endif @@ -60,15 +60,18 @@ prepack: $(BINARY_OUTPUT_FOLDER)/version cd $(BINARY_WRAPPER_DIR); npm version "$(shell cat $(WORKING_DIR)/$(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --include-workspace-root npx ts-node ./release-scripts/prune-dependencies-in-packagejson.ts +.PHONY: clean-package-files +clean-package-files: + @git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json $(BINARY_WRAPPER_DIR)/package.json $(BINARY_WRAPPER_DIR)/package-lock.json 2> /dev/null + .PHONY: clean-prepack -clean-prepack: - git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json $(BINARY_WRAPPER_DIR)/package.json $(BINARY_WRAPPER_DIR)/package-lock.json - rm -f prepack +clean-prepack: clean-package-files + @rm -f prepack .PHONY: clean-ts clean-ts: - npm run clean - rm -f -r $(BINARY_RELEASES_FOLDER_TS_CLI) + @npm run clean + @rm -f -r $(BINARY_RELEASES_FOLDER_TS_CLI) $(BINARY_OUTPUT_FOLDER)/sha256sums.txt.asc: ./release-scripts/sha256sums.txt.asc.sh @@ -199,9 +202,15 @@ test-binary-wrapper: build-binary-wrapper .PHONY: pre-build pre-build: pre-build-binary-wrapper $(BINARY_RELEASES_FOLDER_TS_CLI) +.PHONY: build-fips +build-fips: pre-build + @cd $(EXTENSIBLE_CLI_DIR); $(MAKE) fips build-full install bindir=$(WORKING_DIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1 + @$(MAKE) clean-package-files + .PHONY: build build: pre-build @cd $(EXTENSIBLE_CLI_DIR); $(MAKE) build-full install bindir=$(WORKING_DIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1 + @$(MAKE) clean-package-files .PHONY: sign sign: @@ -209,8 +218,12 @@ sign: .PHONY: clean clean: - @cd $(EXTENSIBLE_CLI_DIR); $(MAKE) clean-full - $(MAKE) clean-prepack + @cd $(EXTENSIBLE_CLI_DIR); $(MAKE) clean-full USE_LEGACY_EXECUTABLE_NAME=1 + @$(MAKE) clean-prepack + +.PHONY: clean-golang +clean-golang: + @cd $(EXTENSIBLE_CLI_DIR); $(MAKE) clean USE_LEGACY_EXECUTABLE_NAME=1 # targets responsible for the testing of CLI build .PHONY: acceptance-test-with-proxy diff --git a/cliv2/Makefile b/cliv2/Makefile index 97a009fb56..9c2cdba640 100644 --- a/cliv2/Makefile +++ b/cliv2/Makefile @@ -5,13 +5,14 @@ GOOS = $(shell go env GOOS) GOARCH = $(shell go env GOARCH) GOHOSTOS = $(shell go env GOHOSTOS) GOHOSTARCH = $(shell go env GOHOSTARCH) +FIPS_CRYPTO_BACKEND_DEFAULT = opensslcrypto +FIPS_CRYPTO_BACKEND = HASH = sha HASH_ALGORITHM = 256 -CLI_V2_VERSION_TAG = -CLI_V1_VERSION_TAG = +CLI_V1_VERSION_TAG = 0.0.0 CLI_V1_LOCATION = USE_LEGACY_EXECUTABLE_NAME = -LDFLAGS = -s -w -X main.InternalOS=$(GOOS) +LDFLAGS = -s -w SHASUM_CMD = shasum # Make directories per convention @@ -33,12 +34,13 @@ _GO_OS = $(GOOS) _V1_OS = $(GOOS) _V1_ARCH = $(GOARCH) _EMPTY = -_EXE_POSTFIX = +_EXE_POSTFIX = +_FIPS_EXTENSION = fips _SEPARATOR = _ ifeq ($(_V1_OS), darwin) _V1_OS = macos -#temporarily irgnoring any architecture for macos v1 binaries, this will enable to natively compile v2 as darwin arm64 and bundle v1 as amd64 +#temporarily ignoring any architecture for macos v1 binaries, this will enable to natively compile v2 as darwin arm64 and bundle v1 as amd64 _V1_ARCH = else ifeq ($(_V1_OS), windows) _V1_OS = win @@ -55,15 +57,6 @@ else V1_PLATFORM_STING = $(_V1_OS)-$(_V1_ARCH) endif -# find out whether to download CLIv1 executable -_V1_DOWNLOAD = true -ifneq ($(CLI_V1_LOCATION), $(_EMPTY)) - ifeq ($(CLI_V1_VERSION_TAG), $(_EMPTY)) - CLI_V1_VERSION_TAG = $(shell cat "$(CLI_V1_LOCATION)/version") - endif - _V1_DOWNLOAD = false -endif - ifeq ($(_GO_OS), alpine) _GO_OS = linux endif @@ -76,20 +69,18 @@ V2_EXECUTABLE_NAME = $(APPLICATION_NAME)$(_SEPARATOR)$(V2_PLATFORM_STRING)$(_EXE ifneq ($(USE_LEGACY_EXECUTABLE_NAME), $(_EMPTY)) V2_EXECUTABLE_NAME = $(V1_EXECUTABLE_NAME) + _SEPARATOR = - endif V1_EXECUTABLE_NAME = $(APPLICATION_NAME)-$(V1_PLATFORM_STING)$(_EXE_POSTFIX) V2_DIRECTORY = $(WORKING_DIR)/internal/cliv2 V1_DIRECTORY = $(WORKING_DIR)/internal/embedded/cliv1 -V1_DOWNLOAD_LINK = https://static.snyk.io/cli/v$(CLI_V1_VERSION_TAG)/$(V1_EXECUTABLE_NAME) V1_EMBEDDED_FILE_TEMPLATE = $(V1_DIRECTORY)/embedded_binary_template.txt V1_EMBEDDED_FILE_OUTPUT = embedded$(_SEPARATOR)$(V2_PLATFORM_STRING).go V1_WORKING_DIR = $(WORKING_DIR)/.. V1_BUILD_TYPE = build:prod V1_BINARY_FOLDER = ts-cli-binaries HASH_STRING = $(HASH)$(HASH_ALGORITHM) -TEST_SNYK_EXECUTABLE_PATH=$(BUILD_DIR)/$(V2_EXECUTABLE_NAME) -TEST_EXECUTABLE_NAME = $(TEST_NAME)$(_SEPARATOR)$(V2_PLATFORM_STRING)$(_EXE_POSTFIX) SIGN_SCRIPT = $(WORKING_DIR)/scripts/sign_$(_GO_OS).sh ISSIGNED_SCRIPT = $(WORKING_DIR)/scripts/issigned_$(_GO_OS).sh EMBEDDED_DATA_DIR = $(WORKING_DIR)/internal/embedded/_data @@ -101,18 +92,13 @@ ifeq ($(GOHOSTOS), windows) ISSIGNED_SCRIPT = $(SPECIAL_SHELL) $(WORKING_DIR)/scripts/issigned_$(GOHOSTOS).ps1 endif +ifeq ($(_GO_OS), windows) + FIPS_CRYPTO_BACKEND_DEFAULT = cngcrypto +endif + # some make file variables LOG_PREFIX = -- -# determine the latest cli version if no version was explicitly defined -$(CACHE_DIR)/version.mk: $(CACHE_DIR) -ifeq ($(CLI_V1_VERSION_TAG), $(_EMPTY)) - $(eval CLI_V1_VERSION_TAG := $(shell curl --fail --progress-bar -L https://static.snyk.io/cli/latest/version)) -endif - @printf "CLI_V1_VERSION_TAG=$(CLI_V1_VERSION_TAG)\nCLI_V2_VERSION_TAG=$(CLI_V2_VERSION_TAG)" > $(CACHE_DIR)/version.mk - @echo "$(LOG_PREFIX) Using cliv1 version ( $(CLI_V1_VERSION_TAG) )" - @echo "$(LOG_PREFIX) Building cliv2 version ( $(CLI_V2_VERSION_TAG) )" - $(BUILD_DIR): @mkdir $@ @@ -120,35 +106,19 @@ $(CACHE_DIR): @mkdir $@ $(CACHE_DIR)/variables.mk: $(CACHE_DIR) - @printf "GOOS=$(GOOS)\nGOARCH=$(GOARCH)\n" > $(CACHE_DIR)/variables.mk + @printf "GOOS=$(GOOS)\nGOARCH=$(GOARCH)\nUSE_LEGACY_EXECUTABLE_NAME=$(USE_LEGACY_EXECUTABLE_NAME)\n" > $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT): @echo "$(LOG_PREFIX) Generating ( $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) )" @sed -e 's/FILENAME/$(V1_EXECUTABLE_NAME)/g' $(V1_EMBEDDED_FILE_TEMPLATE) > $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) -$(V1_DIRECTORY)/cliv1.version: - @echo "$(CLI_V1_VERSION_TAG)" > $(V1_DIRECTORY)/cliv1.version - -$(V2_DIRECTORY)/cliv2.version: - @echo "$(CLI_V2_VERSION_TAG)" > $(V2_DIRECTORY)/cliv2.version - -$(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME): $(V1_DIRECTORY)/cliv1.version -ifeq ($(_V1_DOWNLOAD), true) - @echo "$(LOG_PREFIX) Downloading cliv1 executable ( $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) )" - @curl --fail --progress-bar -Lo $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) $(V1_DOWNLOAD_LINK) -else +$(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME): @echo "$(LOG_PREFIX) Copying cliv1 executable ( $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) )" @cp $(CLI_V1_LOCATION)/$(V1_EXECUTABLE_NAME) $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) -endif $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING): $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) -ifeq ($(_V1_DOWNLOAD), true) - @echo "$(LOG_PREFIX) Downloading cliv1 checksum ( $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) )" - @curl --fail --progress-bar -Lo $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) $(V1_DOWNLOAD_LINK).$(HASH_STRING) -else @echo "$(LOG_PREFIX) Copying cliv1 checksum ( $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) )" @cp $(CLI_V1_LOCATION)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) -endif .PHONY: _validate_sha_v1 _validate_sha_v1: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) @@ -159,13 +129,35 @@ _validate_sha_v1: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) .PHONY: dependencies dependencies: $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME) $(V1_DIRECTORY)/$(V1_EXECUTABLE_NAME).$(HASH_STRING) _validate_sha_v1 +.PHONY: summary +summary: + @echo "$(LOG_PREFIX) Build Summary" + @echo "$(LOG_PREFIX) Using Go installation: $(shell which go)" + @echo "$(LOG_PREFIX) Non-Standard Crypto-Backend: $(FIPS_CRYPTO_BACKEND)" + @echo "$(LOG_PREFIX) Building Binary: $(BUILD_DIR)/$(V2_EXECUTABLE_NAME)" + @echo "$(LOG_PREFIX) Building version: $(CLI_V1_VERSION_TAG)" + + # prepare the workspace and cache global parameters .PHONY: configure -configure: $(V2_DIRECTORY)/cliv2.version $(CACHE_DIR) $(CACHE_DIR)/version.mk $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses +configure: summary $(CACHE_DIR) $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses $(BUILD_DIR)/$(V2_EXECUTABLE_NAME): $(BUILD_DIR) $(SRCS) @echo "$(LOG_PREFIX) Building ( $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) )" - @GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go + @GOEXPERIMENT=$(FIPS_CRYPTO_BACKEND) GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -tags=application -ldflags="$(LDFLAGS) -X main.internalOS=$(GOOS) -X github.com/snyk/cli/cliv2/internal/embedded/cliv1.snykCLIVersion=$(CLI_V1_VERSION_TAG)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go + +.PHONY: fips +fips: +# adapt executable name if fips is used +ifneq (, $(_EXE_POSTFIX)) + $(eval V2_EXECUTABLE_NAME := $(subst $(_EXE_POSTFIX),$(_SEPARATOR)$(_FIPS_EXTENSION)$(_EXE_POSTFIX), $(V2_EXECUTABLE_NAME))) +else + $(eval V2_EXECUTABLE_NAME := "$(V2_EXECUTABLE_NAME)$(_SEPARATOR)$(_FIPS_EXTENSION)") +endif + +# assign name of crypto backend to use + $(eval FIPS_CRYPTO_BACKEND := $(FIPS_CRYPTO_BACKEND_DEFAULT)) + @echo "$(LOG_PREFIX) FIPS" .PHONY: build build: configure $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) @@ -179,13 +171,13 @@ $(WORKING_DIR)/internal/httpauth/generated/spnego_generated_mock.go: $(CACHE_DIR)/prepare-3rd-party-licenses: @echo "$(LOG_PREFIX) Preparing 3rd Party Licenses" - @GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) scripts/prepare_licenses.sh > $(CACHE_DIR)/prepare-3rd-party-licenses + @GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) scripts/prepare_licenses.sh > $(CACHE_DIR)/prepare-3rd-party-licenses 2> /dev/null .PHONY: generate generate: $(WORKING_DIR)/internal/httpauth/generated/httpauth_generated_mock.go $(WORKING_DIR)/internal/httpauth/generated/spnego_generated_mock.go .PHONY: whiteboxtest -whiteboxtest: +whiteboxtest: @echo "$(LOG_PREFIX) Running $@" @$(GOCMD) test -cover ./... @@ -234,8 +226,7 @@ clean-ts-cli: # build the full CLI (Typescript+Golang) .PHONY: build-full -build-full: | build-ts-cli - @$(MAKE) build CLI_V1_VERSION_TAG=$(CLI_V1_VERSION_TAG) CLI_V1_LOCATION="$(CLI_V1_LOCATION)" +build-full: build-ts-cli build # clean the full CLI (Typescript+Golang) .PHONY: clean-full @@ -246,18 +237,17 @@ all: build-full .PHONY: clean clean: + @echo "$(LOG_PREFIX) Cleaning ( $(DESTDIR)$(bindir)/$(V2_EXECUTABLE_NAME) )" @$(GOCMD) clean @rm -f -r $(BUILD_DIR) @rm -f -r $(CACHE_DIR) @rm -f $(V1_DIRECTORY)/$(APPLICATION_NAME)-* @rm -f $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) - @rm -f $(V1_DIRECTORY)/cliv1.version - @rm -f $(V2_DIRECTORY)/cliv2.version @rm -f -r $(EMBEDDED_DATA_DIR)/licenses .PHONY: install install: - @echo "$(LOG_PREFIX) Installing $(V2_EXECUTABLE_NAME) ( $(DESTDIR)$(bindir) $(CLI_V1_VERSION_TAG))" + @echo "$(LOG_PREFIX) Installing ( $(DESTDIR)$(bindir)/$(V2_EXECUTABLE_NAME) )" @mkdir -p $(DESTDIR)$(bindir) @rm -f $(DESTDIR)$(bindir)/$(V2_EXECUTABLE_NAME) @cp $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(DESTDIR)$(bindir) @@ -267,6 +257,7 @@ help: @echo "\n Main targets (Golang only):" @echo "$(LOG_PREFIX) lint" @echo "$(LOG_PREFIX) format" + @echo "$(LOG_PREFIX) fips" @echo "$(LOG_PREFIX) build" @echo "$(LOG_PREFIX) sign" @echo "$(LOG_PREFIX) test" @@ -279,9 +270,7 @@ help: @echo "\nAvailable parameter:" @echo "$(LOG_PREFIX) GOOS Specify Operating System to compile for (see golang GOOS, default=$(GOOS))" @echo "$(LOG_PREFIX) GOARCH Specify Architecture to compile for (see golang GOARCH, default=$(GOARCH))" - @echo "$(LOG_PREFIX) CLI_V2_VERSION_TAG Version of the CLIv2 without the CLIv1 version (default=$(CLI_V2_VERSION_TAG))" @echo "$(LOG_PREFIX) CLI_V1_VERSION_TAG Version of the CLIv1 to bundle" @echo "$(LOG_PREFIX) CLI_V1_LOCATION Filesystem location of CLIv1 binaries to bundle, if specified, CLI_V1_VERSION_TAG is also required" - @echo "$(LOG_PREFIX) TEST_SNYK_EXECUTABLE_PATH Filesystem location of binary under test (default=$(TEST_SNYK_EXECUTABLE_PATH))" - @echo "$(LOG_PREFIX) prefix Installation prefix (default=$(prefix))" + @echo "$(LOG_PREFIX) prefix Installation prefix (default=$(prefix))" @echo "$(LOG_PREFIX) DESTDIR For staged installations" diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index 31d6d17791..0554c790a0 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -32,7 +32,7 @@ import ( "github.com/spf13/pflag" ) -var InternalOS string +var internalOS string var engine workflow.Engine var config configuration.Configuration var helpProvided bool @@ -386,7 +386,7 @@ func writeLogHeader(config configuration.Configuration, networkAccess networking } tablePrint("Version", cliv2.GetFullVersion()) - tablePrint("Platform", InternalOS+" "+runtime.GOARCH) + tablePrint("Platform", internalOS+" "+runtime.GOARCH) tablePrint("API", config.GetString(configuration.API_URL)) tablePrint("Cache", config.GetString(configuration.CACHE_PATH)) tablePrint("Organization", org) @@ -450,7 +450,7 @@ func MainWithErrorCode() int { networking.UserAgent( networking.UaWithConfig(config), networking.UaWithApplication("snyk-cli", cliv2.GetFullVersion()), - networking.UaWithOS(InternalOS)).String(), + networking.UaWithOS(internalOS)).String(), ) if debugEnabled { @@ -461,7 +461,7 @@ func MainWithErrorCode() int { cliAnalytics := engine.GetAnalytics() cliAnalytics.SetVersion(cliv2.GetFullVersion()) cliAnalytics.SetCmdArguments(os.Args[1:]) - cliAnalytics.SetOperatingSystem(InternalOS) + cliAnalytics.SetOperatingSystem(internalOS) if config.GetBool(configuration.ANALYTICS_DISABLED) == false { defer sendAnalytics(cliAnalytics, debugLogger) } diff --git a/cliv2/internal/cliv2/cliv2.go b/cliv2/internal/cliv2/cliv2.go index 0b3a0add27..8e72f32e2c 100644 --- a/cliv2/internal/cliv2/cliv2.go +++ b/cliv2/internal/cliv2/cliv2.go @@ -45,9 +45,6 @@ const ( V2_ABOUT Handler = iota ) -//go:embed cliv2.version -var version_prefix string - func NewCLIv2(cacheDirectory string, debugLogger *log.Logger) (*CLI, error) { v1BinaryLocation, err := cliv1.GetFullCLIV1TargetPath(cacheDirectory) if err != nil { @@ -182,13 +179,7 @@ func (c *CLI) ExtractV1Binary() error { func GetFullVersion() string { v1Version := cliv1.CLIV1Version() - v2Version := strings.TrimSpace(version_prefix) - - if len(v2Version) > 0 { - return v2Version + "." + v1Version - } else { - return v1Version - } + return v1Version } func (c *CLI) GetIntegrationName() string { diff --git a/cliv2/internal/cliv2/cliv2_test.go b/cliv2/internal/cliv2/cliv2_test.go index bf90ada2e7..ad3b7c6b26 100644 --- a/cliv2/internal/cliv2/cliv2_test.go +++ b/cliv2/internal/cliv2/cliv2_test.go @@ -162,9 +162,7 @@ func Test_prepareV1Command(t *testing.T) { assert.Nil(t, err) } -func Test_executeRunV1(t *testing.T) { - expectedReturnCode := 0 - +func Test_extractOnlyOnce(t *testing.T) { cacheDir := getCacheDir(t) tmpDir := utils.GetTemporaryDirectory(cacheDir, cliv2.GetFullVersion()) @@ -175,8 +173,7 @@ func Test_executeRunV1(t *testing.T) { // run once assert.Nil(t, cli.Init()) - actualReturnCode := cliv2.DeriveExitCode(cli.Execute(getProxyInfoForTest(), []string{"--help"})) - assert.Equal(t, expectedReturnCode, actualReturnCode) + cli.Execute(getProxyInfoForTest(), []string{"--help"}) assert.FileExists(t, cli.GetBinaryLocation()) fileInfo1, _ := os.Stat(cli.GetBinaryLocation()) @@ -185,8 +182,7 @@ func Test_executeRunV1(t *testing.T) { // run twice assert.Nil(t, cli.Init()) - actualReturnCode = cliv2.DeriveExitCode(cli.Execute(getProxyInfoForTest(), []string{"--help"})) - assert.Equal(t, expectedReturnCode, actualReturnCode) + cli.Execute(getProxyInfoForTest(), []string{"--help"}) assert.FileExists(t, cli.GetBinaryLocation()) fileInfo2, _ := os.Stat(cli.GetBinaryLocation()) @@ -221,10 +217,6 @@ func Test_init_extractDueToInvalidBinary(t *testing.T) { // execute to test that the cli can run successfully assert.FileExists(t, cli.GetBinaryLocation()) - // prove that we now can execute the invalid binary - _, binError = exec.Command(cli.GetBinaryLocation(), "--help").Output() - assert.Nil(t, binError) - fileInfo2, _ := os.Stat(cli.GetBinaryLocation()) assert.NotEqual(t, fileInfo1.ModTime(), fileInfo2.ModTime()) @@ -248,26 +240,6 @@ func Test_executeRunV2only(t *testing.T) { } -func Test_executeEnvironmentError(t *testing.T) { - expectedReturnCode := 0 - - cacheDir := getCacheDir(t) - tmpDir := utils.GetTemporaryDirectory(cacheDir, cliv2.GetFullVersion()) - - assert.NoDirExists(t, tmpDir) - - // fill Environment Variable - _ = os.Setenv(constants.SNYK_INTEGRATION_NAME_ENV, "someName") - - // create instance under test - cli, _ := cliv2.NewCLIv2(cacheDir, discardLogger) - assert.Nil(t, cli.Init()) - - actualReturnCode := cliv2.DeriveExitCode(cli.Execute(getProxyInfoForTest(), []string{"--help"})) - assert.Equal(t, expectedReturnCode, actualReturnCode) - assert.FileExists(t, cli.GetBinaryLocation()) -} - func Test_executeUnknownCommand(t *testing.T) { expectedReturnCode := constants.SNYK_EXIT_CODE_ERROR diff --git a/cliv2/internal/embedded/cliv1/cliv1.go b/cliv2/internal/embedded/cliv1/cliv1.go index 70514fa0c4..fa6b3a924e 100644 --- a/cliv2/internal/embedded/cliv1/cliv1.go +++ b/cliv2/internal/embedded/cliv1/cliv1.go @@ -9,8 +9,8 @@ import ( "github.com/snyk/cli/cliv2/internal/utils" ) -//go:embed cliv1.version -var snykCLIVersion string +// The actual version gets injected at build time +var snykCLIVersion string = "0.0.0" func CLIV1Version() string { return strings.TrimSpace(snykCLIVersion) diff --git a/cliv2/internal/embedded/cliv1/dummy_embedded_legacy_cli.go b/cliv2/internal/embedded/cliv1/dummy_embedded_legacy_cli.go new file mode 100644 index 0000000000..172016b75e --- /dev/null +++ b/cliv2/internal/embedded/cliv1/dummy_embedded_legacy_cli.go @@ -0,0 +1,22 @@ +//go:build !application + +// For tests + +package cliv1 + +import ( + _ "embed" +) + +var snykCLIBytes []byte = []byte("\n") + +func getCLIv1Filename() string { + return "FILENAME" +} + +var snykCLISHA256 string = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + +func ExpectedSHA256() string { + sha256 := snykCLISHA256[0:64] + return sha256 +} diff --git a/cliv2/internal/embedded/cliv1/embedded_binary_template.txt b/cliv2/internal/embedded/cliv1/embedded_binary_template.txt index 6d1408892b..63a6cdf253 100644 --- a/cliv2/internal/embedded/cliv1/embedded_binary_template.txt +++ b/cliv2/internal/embedded/cliv1/embedded_binary_template.txt @@ -1,3 +1,4 @@ +//go:build application package cliv1 import ( diff --git a/scripts/download_go.py b/scripts/download_go.py new file mode 100644 index 0000000000..a2da27a101 --- /dev/null +++ b/scripts/download_go.py @@ -0,0 +1,150 @@ +import argparse +import hashlib +import os +import platform +import urllib.request +import tarfile +import zipfile +import tempfile +import sys + +# determine go binary to download and extract +def get_go_binary_name(go_os, go_arch, go_version): + filename = "go" + go_version + "." + go_os + "-" + go_arch + + # determine file extension + if go_os == "windows": + filename += ".zip" + else: + filename += ".tar.gz" + + print("Binary to download: " + filename) + return filename + +# calculate sha256sum of file +def calculate_sha256sum(file): + print("Calculating sha256sum of file: " + file) + + try: + with open(file, "rb") as f: + sha256sum = hashlib.sha256(f.read()).hexdigest() + except FileNotFoundError as e: + print("Error while calculating sha256sum of file:", e) + sys.exit(1) + + return sha256sum + +# extract sha256sum from file contents +def extract_sha256sum_from_file(file): + print("Extracting sha256sum from file: " + file) + + with open(file, "r") as f: + first_line = f.readline().strip() # Read the first line and remove leading/trailing whitespace + first_word = first_line.split()[0] # Sp + + return first_word + +# download a file +def download_file(download_path, filename, base_url): + url = base_url + filename + file_path = os.path.join(download_path, filename) + + print("Downloading from: " + url + " to: " + file_path) + + try: + urllib.request.urlretrieve(url, file_path) + print("Download complete") + except urllib.error.URLError as e: + print("Error while downloading the file:", e) + sys.exit(1) + + return file_path + +# extract tar file +def extract_tar(tar_gz_file, extract_path): + print("Extracting: " + tar_gz_file + " to: " + extract_path) + try: + with tarfile.open(tar_gz_file, "r:gz") as tar: + tar.extractall(path=extract_path) + print("Extraction complete") + except tarfile.TarError as e: + print("Error while extracting the tar file: ", e) + sys.exit(1) + +# unzip zip file +def unzip_file(zip_file, extract_path): + print("Unzipping: " + zip_file + " to: " + extract_path) + try: + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(extract_path) + print("Unzipping complete") + except zipfile.BadZipFile as e: + print("Error while unzipping the file: ", e) + sys.exit(1) + +# setup argparse +def init_argparse(): + parser = argparse.ArgumentParser( + prog="download_go", + description="Download and install a specific version of Go." + ) + parser.add_argument("version", help="Version of Go to download (e.g., 1.20)") + parser.add_argument("--go_os", + help="OS to download for (e.g., linux, windows, darwin)", + choices=["linux", "windows", "darwin"], + required=True) + parser.add_argument("--go_arch", + help="Architecture to download for (e.g., amd64, arm64)", + choices=["amd64", "arm64", "armv6l"], + required=True) + parser.add_argument("--base_url", + help="Base URL to download from (e.g., https://storage.googleapis.com/golang/)", + default="https://storage.googleapis.com/golang/") + parser.add_argument("--extraction_path", + help="Path to download the file to (e.g., /tmp)", + default=os.getcwd()) + + return parser.parse_args() + +if __name__ == "__main__": + args = init_argparse() + + # check if go is cached, before trying to download and extract it + file_dir = os.path.join(os.path.abspath(args.extraction_path), "go") + if os.path.exists(file_dir): + print("Restored from cache, skipping download and extraction") + sys.exit(0) + + # create temporary download path + download_path = tempfile.gettempdir() + + # determine the go binary to download + binary_name = get_go_binary_name(args.go_os, args.go_arch, args.version) + + # download the go binary + binary_file = download_file(download_path, binary_name, args.base_url) + # get the sha256sum of the downloaded file + binary_file_sha256 = calculate_sha256sum(binary_file) + + # download the sha256sum file for the go binary + sha256sum_file = download_file(download_path, binary_name + ".sha256", args.base_url) + + # get the expected sha256sum for the downloaded file + expected_sha256sum = extract_sha256sum_from_file(sha256sum_file) + + # compare the expected sha256sum with the actual sha256sum of the downloaded file + if expected_sha256sum == binary_file_sha256: + print("sha256sum check passed.") + print(" Actual: " + binary_file_sha256) + print("Expected: " + expected_sha256sum) + else: + print("sha256sum check failed.") + print(" Actual: " + binary_file_sha256) + print("Expected: " + expected_sha256sum) + sys.exit(1) + + # extract the downloaded file + if binary_name.endswith(".zip"): + unzip_file(binary_file, os.path.abspath(args.extraction_path)) + else: + extract_tar(binary_file, os.path.abspath(args.extraction_path))