feat: OpAMP Package Update functionality

.github/dependabot.yml

@@ -45,3 +45,21 @@ updates:
     commit-message:
       prefix: "deps"
       include: "scope"
+  - package-ecosystem: "gomod"
+    directory: "/updater"
+    schedule:
+      interval: "weekly"
+    ignore:
+      # Opentelemetry updates will be done manually
+      - dependency-name: "github.com/open-telemetry/opentelemetry-collector*"
+      - dependency-name: "go.opentelemetry.io/collector/*"
+    commit-message:
+      prefix: "deps"
+      include: "scope"
+  - package-ecosystem: "gomod"
+    directory: "/packagestate"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "deps"
+      include: "scope"

.github/workflows/manual_msi_build.yml

@@ -36,8 +36,10 @@ jobs:
           args: build --single-target --skip-validate --rm-dist --snapshot
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Copy Windows Binary
+      - name: Copy Windows Collector Binary
         run: cp dist/collector_windows_amd64_v1/observiq-otel-collector.exe windows/observiq-otel-collector.exe
+      - name: Copy Windows Updater Binary
+        run: cp dist/updater_windows_amd64_v1/updater.exe windows/updater.exe
       - name: Copy Plugins to MSI Build Directory
         run: cp -r release_deps/plugins windows/
       - name: Copy Example Config

.github/workflows/multi_build.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Scan Third Party Dependency Licenses
         run: |
           go install github.com/uw-labs/lichen@v0.1.5
-          lichen --config=./license.yaml $(find dist/collector_*)
+          lichen --config=./license.yaml $(find dist/collector_* dist/updater_*)
   build_darwin:
     runs-on: macos-11
     steps:
@@ -55,7 +55,7 @@ jobs:
       - name: Scan Third Party Dependency Licenses
         run: |
           go install github.com/uw-labs/lichen@v0.1.5
-          lichen --config=./license.yaml $(find dist/collector_*)
+          lichen --config=./license.yaml $(find dist/collector_* dist/updater_*)
   build_windows:
     runs-on: windows-2019
     steps:
@@ -80,4 +80,4 @@ jobs:
       - name: Scan Third Party Dependency Licenses
         run: |
           go install github.com/uw-labs/lichen@v0.1.5
-          lichen --config=./license.yaml dist/collector_windows_amd64.exe
+          lichen --config=./license.yaml dist/collector_windows_amd64.exe dist/updater_windows_amd64.exe

.github/workflows/release.yml

@@ -33,8 +33,10 @@ jobs:
           args: build --single-target --skip-validate --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Copy Windows Binary
+      - name: Copy Windows Collector Binary
         run: cp dist/collector_windows_amd64_v1/observiq-otel-collector.exe windows/observiq-otel-collector.exe
+      - name: Copy Windows Updater Binary
+        run: cp dist/updater_windows_amd64_v1/updater.exe windows/updater.exe
       - name: Copy Plugins to MSI Build Directory
         run: cp -r release_deps/plugins windows/
       - name: Copy Example Config

.github/workflows/tests.yml

@@ -51,3 +51,9 @@ jobs:
             ${{ runner.os }}-go-
       - name: Run Tests
         run: make test
+      - name: Run Updater Integration Tests (non-linux)
+        if: matrix.os != 'ubuntu-20.04'
+        run: make test-updater-integration
+      - name: Run Updater Integration Tests (linux)
+        if: matrix.os == 'ubuntu-20.04'
+        run: sudo make test-updater-integration

.gitignore

@@ -10,7 +10,7 @@ cosign.*
 gpg.*
 *.msi
 *.exe
-*.zip
+windows/*.zip
 windows/**/wix.dynamic.json
 windows/**/wix
 windows/config.yaml
@@ -20,7 +20,7 @@ windows/plugins
 opentelemetry-java-contrib-jmx-metrics.jar
 VERSION.txt
 release_deps
-tmp
+/tmp
 
 # OpAmp Files
 collector.yaml

.goreleaser.yml

@@ -31,6 +31,32 @@ builds:
       - -X github.com/observiq/observiq-otel-collector/internal/version.gitHash={{ .FullCommit }}
       - -X github.com/observiq/observiq-otel-collector/internal/version.date={{ .Date }}
     no_unique_dist_dir: false
+  - id: updater
+    binary: updater
+    dir: ./updater/
+    main: ./cmd/updater
+    env:
+      - CGO_ENABLED=0
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    goos:
+      - windows
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+      - arm
+    ignore:
+      - goos: windows
+        goarch: arm
+      - goos: windows
+        goarch: arm64
+    ldflags:
+      - -s -w
+      - -X github.com/observiq/observiq-otel-collector/updater/internal/version.version=v{{ .Version }}
+      - -X github.com/observiq/observiq-otel-collector/updater/internal/version.gitHash={{ .FullCommit }}
+      - -X github.com/observiq/observiq-otel-collector/updater/internal/version.date={{ .Date }}
+    no_unique_dist_dir: false
 
 # https://goreleaser.com/customization/archive/
 archives:
@@ -56,7 +82,12 @@ archives:
       - src: release_deps/com.observiq.collector.plist
         dst: "install"
         strip_parent: true
-
+      - src: release_deps/windows_service.json
+        dst: install
+        strip_parent: true
+      - src: release_deps/observiq-otel-collector.service
+        dst: "install"
+        strip_parent: true
     format_overrides:
       - goos: windows
         format: zip

Makefile

@@ -10,6 +10,8 @@ OUTDIR=./dist
 GOOS ?= $(shell go env GOOS)
 GOARCH ?= $(shell go env GOARCH)
 
+INTEGRATION_TEST_ARGS?=-tags integration
+
 ifeq ($(GOOS), windows)
 EXT?=.exe
 else
@@ -21,11 +23,23 @@ CURRENT_TAG := $(shell git tag --sort=v:refname --points-at HEAD | grep -E "v[0-
 # Version will be the tag pointing to the current commit, or the previous version tag if there is no such tag
 VERSION ?= $(if $(CURRENT_TAG),$(CURRENT_TAG),$(PREVIOUS_TAG))
 
-# Default build target; making this should build for the current os/arch
+# Build binaries for current GOOS/GOARCH by default
+.DEFAULT_GOAL := build-binaries
+
+# Builds just the collector for current GOOS/GOARCH pair
 .PHONY: collector
 collector:
 	go build -ldflags "-s -w -X github.com/observiq/observiq-otel-collector/internal/version.version=$(VERSION)" -o $(OUTDIR)/collector_$(GOOS)_$(GOARCH)$(EXT) ./cmd/collector
 
+# Builds just the updater for current GOOS/GOARCH pair
+.PHONY: updater
+updater:
+	cd ./updater/; go build -ldflags "-s -w -X github.com/observiq/observiq-otel-collector/internal/version.version=$(VERSION)" -o ../$(OUTDIR)/updater_$(GOOS)_$(GOARCH)$(EXT) ./cmd/updater
+
+# Builds the updater + collector for current GOOS/GOARCH pair
+.PHONY: build-binaries
+build-binaries: collector updater
+
 .PHONY: build-all
 build-all: build-linux build-darwin build-windows
 
@@ -40,27 +54,27 @@ build-windows: build-windows-amd64
 
 .PHONY: build-linux-amd64
 build-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) collector
+	GOOS=linux GOARCH=amd64 $(MAKE) build-binaries -j2
 
 .PHONY: build-linux-arm64
 build-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) collector
+	GOOS=linux GOARCH=arm64 $(MAKE) build-binaries -j2
 
 .PHONY: build-linux-arm
 build-linux-arm:
-	GOOS=linux GOARCH=arm $(MAKE) collector
+	GOOS=linux GOARCH=arm $(MAKE) build-binaries -j2
 
 .PHONY: build-darwin-amd64
 build-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) collector
+	GOOS=darwin GOARCH=amd64 $(MAKE) build-binaries -j2
 
 .PHONY: build-darwin-arm64
 build-darwin-arm64:
-	GOOS=darwin GOARCH=arm64 $(MAKE) collector
+	GOOS=darwin GOARCH=arm64 $(MAKE) build-binaries -j2
 
 .PHONY: build-windows-amd64
 build-windows-amd64:
-	GOOS=windows GOARCH=amd64 $(MAKE) collector
+	GOOS=windows GOARCH=amd64 $(MAKE) build-binaries -j2
 
 # tool-related commands
 .PHONY: install-tools
@@ -97,6 +111,10 @@ test-with-cover:
 	$(MAKE) for-all CMD="go test -coverprofile=cover.out ./..."
 	$(MAKE) for-all CMD="go tool cover -html=cover.out -o cover.html"
 
+.PHONY: test-updater-integration 
+test-updater-integration:
+	cd updater; go test $(INTEGRATION_TEST_ARGS) -race ./...
+
 .PHONY: bench
 bench:
 	$(MAKE) for-all CMD="go test -benchmem -run=^$$ -bench ^* ./..."
@@ -115,7 +133,9 @@ tidy:
 
 .PHONY: gosec
 gosec:
-	gosec ./...
+	gosec -exclude-dir updater  ./...
+# exclude the testdata dir; it contains a go program for testing.
+	cd updater; gosec -exclude-dir internal/service/testdata ./...
 
 # This target performs all checks that CI will do (excluding the build itself)
 .PHONY: ci-checks
@@ -157,6 +177,8 @@ release-prep:
 	@cp config/example.yaml release_deps/config.yaml
 	@cp config/logging.yaml release_deps/logging.yaml
 	@cp service/com.observiq.collector.plist release_deps/com.observiq.collector.plist
+	@jq ".files[] | select(.service != null)" windows/wix.json >> release_deps/windows_service.json
+	@cp service/observiq-otel-collector.service release_deps/observiq-otel-collector.service
 
 # Build, sign, and release
 .PHONY: release
@@ -184,7 +206,7 @@ clean:
 
 .PHONY: scan-licenses
 scan-licenses:
-	lichen --config=./license.yaml $$(find dist/collector_* | grep -v 'sig\|json\|CHANGELOG.md\|yaml\|SHA256' | xargs)
+	lichen --config=./license.yaml $$(find dist/collector_* dist/updater_*)
 
 .PHONY: generate
 generate:

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-collector v0.0.3-0.20220711143229-08f2752ed367
 	github.com/google/uuid v1.3.0
 	github.com/observiq/observiq-otel-collector/exporter/googlecloudexporter v1.3.0
+	github.com/observiq/observiq-otel-collector/packagestate v0.0.0
 	github.com/observiq/observiq-otel-collector/processor/resourceattributetransposerprocessor v1.3.0
 	github.com/observiq/observiq-otel-collector/receiver/pluginreceiver v1.3.0
 	github.com/open-telemetry/opamp-go v0.2.0
@@ -162,6 +163,7 @@ require (
 	github.com/alecthomas/participle/v2 v2.0.0-alpha9 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/aliyun/aliyun-log-go-sdk v0.1.37 // indirect
+	github.com/andybalholm/brotli v1.0.1 // indirect
 	github.com/antonmedv/expr v1.9.0 // indirect
 	github.com/apache/thrift v0.16.0 // indirect
 	github.com/armon/go-metrics v0.3.10 // indirect
@@ -190,6 +192,7 @@ require (
 	github.com/docker/docker v20.10.17+incompatible // indirect
 	github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -276,6 +279,7 @@ require (
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/knadh/koanf v1.4.2 // indirect
 	github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b // indirect
 	github.com/leoluk/perflib_exporter v0.1.0 // indirect
@@ -306,6 +310,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20201110005315-f5a5f8086c19 // indirect
+	github.com/nwaples/rardecode v1.1.0 // indirect
 	github.com/observiq/ctimefmt v1.0.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudexporter v0.57.2 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.57.2 // indirect
@@ -391,13 +396,15 @@ require (
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
+	github.com/ulikunitz/xz v0.5.9 // indirect
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
 	github.com/vmware/govmomi v0.28.0 // indirect
 	github.com/vultr/govultr/v2 v2.17.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
@@ -454,6 +461,7 @@ require (
 require (
 	github.com/containerd/containerd v1.6.6 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/mholt/archiver/v3 v3.5.1
 	github.com/shirou/gopsutil/v3 v3.22.7
 	github.com/spf13/cobra v1.5.0 // indirect
 	golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e
@@ -465,4 +473,6 @@ replace github.com/observiq/observiq-otel-collector/receiver/pluginreceiver => .
 
 replace github.com/observiq/observiq-otel-collector/exporter/googlecloudexporter => ./exporter/googlecloudexporter
 
+replace github.com/observiq/observiq-otel-collector/packagestate => ./packagestate
+
 replace github.com/GoogleCloudPlatform/opentelemetry-operations-collector v0.0.3-0.20220711143229-08f2752ed367 => github.com/observIQ/opentelemetry-operations-collector v0.0.3-0.20220804143341-7ae64090f52c