From 107b12d7762b4998fb6f72ac60197536f4f2e520 Mon Sep 17 00:00:00 2001
From: Lee Avital <lee.avital@datadoghq.com>
Date: Thu, 29 Aug 2024 15:32:18 -0400
Subject: [PATCH] Download all dependencies (protoc, gogo)

Adds a `rake setup_gogo` task which will clone gogo into a new toolchains directory (which is git ignored).

On every run of codegen, we'll checkout a pinned version of gogo and rebuild it. This should be pretty fast as long as gogo is already checked out.

This PR also adds a new 'toolchains' location where all dependencies are cached. Before we were jumbling a bunch of dependencies into /tmp.

This led to a weird case where we were accidentally (?) using a new protobuf compiler with a legacy .proto library (for stuff like google.Duration). This is now explicit.

The structure of the new toolchains is described in the Rakefile
---
 .gitignore |   6 ++-
 Rakefile   | 143 ++++++++++++++++++++++++++++++++++++-----------------
 2 files changed, 104 insertions(+), 45 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6d888f88..a94ba100 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,8 @@
 # jetbrains IDE directory
 /.idea/
 
-vendor/
\ No newline at end of file
+vendor/
+
+# build time dependencies that are either downloaded or
+# built during codegen
+toolchains/
diff --git a/Rakefile b/Rakefile
index 829bc440..ac648755 100644
--- a/Rakefile
+++ b/Rakefile
@@ -1,25 +1,71 @@
-#
 # Rakefile for agent-payload
-#
+
+# where all toolchains (protobuf compilers, protobuf plugins,etc) are stored
+toolchain_dir=File.join(File.dirname(__FILE__), "toolchains")
+
+# binaries
+toolchain_bin_dir=File.join(toolchain_dir, "bin")
+
+# include path for the legacy and main protobuf compilers respectively
+toolchain_include_dir=File.join(toolchain_dir, "include", "proto")
+toolchain_legacy_include_dir=File.join(toolchain_dir, "include", "proto_legacy")
 
 # we use an older version of protoc for logs and metrics
 protoc_legacy_version="3.5.1"
-protoc_legacy_binary="/tmp/protoc#{protoc_legacy_version}"
+protoc_legacy_binary=File.join(toolchain_bin_dir, "protoc" + protoc_legacy_version)
 
 protoc_version="21.12"
-protoc_binary="/tmp/protoc#{protoc_version}"
+protoc_binary=File.join(toolchain_bin_dir, "protoc" + protoc_version)
 
-gogo_dir="/tmp/gogo"
-protoc_gen_go_dir="/tmp/protoc-gen-go"
+gogo_tag = "v1.3.2"
+gogo_dir=File.join(toolchain_dir,  "gogo")
+gogo_include = "#{toolchain_dir}/gogo/src:#{toolchain_dir}/gogo/src/github.com/gogo/protobuf/protobuf"
+gogo_bin = File.join(toolchain_bin_dir, "gogo-bin-#{gogo_tag}")
 protoc_jsonschema_version="73d5723"
 
+### toolchains is meant to store a cache of all binary dependencies needed to build the agent-payload.
+### this rakefile will download those dependencies on the fly if needed.
+### the structure of toolchains is as follows:
+### /toolchains
+###     /toolchains/bin -- contains any binaries used for building
+###         /toolchains/bin/protoc21.12 -- the new protobuf compiler
+###         /toolchains/bin/protoc3.5.1 -- the legacy protobuf compiler
+###         /toolchains/bin/protoc-gen-go -- the go code generator for protoc
+###         /toolchains/bin/protoc-gen-XXX -- other protobuf generators (vtproto, java, etc.)
+###     /toolchains/include -- contains any protobuf library files for common types (like proto.Duration)
+###         /toolchains/includes/proto -- protobuf libraries bundled with the new protobuf compiler
+###         /toolchains/includes/proto_legacy -- protobuf libraries bundled with the legacy protobuf compiler
+### =>  /toolchains/gogo -- temp directory used to build the gogo_faster generator
 
 namespace :codegen do
+  task :clean do
+      sh "rm -rf #{gogo_dir}"
+      sh "rm -rf #{toolchain_dir}"
+  end
 
-  task :install_protoc do
-    if `bash -c "protoc --version"` != "libprotoc ${protoc_legacy_version}"
+  task :setup_gogo => ['install_protoc_all'] do
+    if ! Dir.exist?(gogo_dir) then
+      directory "#{gogo_dir}/src/github.com/gogo"
+      sh "git clone https://github.com/gogo/protobuf.git #{gogo_dir}/src/github.com/gogo/protobuf"
+    else
+      puts "gogo already cloned into #{gogo_dir}"
+    end
+
+    Dir.chdir("#{gogo_dir}/src/github.com/gogo/protobuf") do
+      sh "git checkout #{gogo_tag}"
+      sh "PATH=$PATH:/tmp GOBIN=#{toolchain_bin_dir} GOPATH=#{gogo_dir} make clean install"
+    end
+  end
+
+
+  task 'install_protoc_all' => ['install_protoc', 'install_protoc_legacy']
+
+  task :install_protoc_legacy  do
+    if `bash -c "#{protoc_legacy_binary} --version"` != "libprotoc ${protoc_version}"
       sh <<-EOF
         /bin/bash <<BASH
+
+        set -euo pipefail
         if [ ! -f #{protoc_legacy_binary} ] ; then
           echo "Downloading protoc #{protoc_legacy_version}"
           cd /tmp
@@ -28,15 +74,27 @@ namespace :codegen do
           else
             curl -OL https://github.com/google/protobuf/releases/download/v#{protoc_legacy_version}/protoc-#{protoc_legacy_version}-linux-x86_64.zip
           fi
-          unzip protoc-#{protoc_legacy_version}*.zip
+          mkdir -p #{toolchain_bin_dir}
+          rm -rf include/ # make sure there is no stale include
+          unzip -o protoc-#{protoc_legacy_version}*.zip
           mv bin/protoc #{protoc_legacy_binary}
+
+          mkdir -p #{toolchain_legacy_include_dir}
+          rm -rf #{toolchain_legacy_include_dir}/**
+          mv include/* #{toolchain_legacy_include_dir}/
         fi
 BASH
       EOF
     end
-    if `bash -c "protoc --version"` != "libprotoc ${protoc_version}"
+  end
+
+
+  task :install_protoc  do
+    if `bash -c "#{protoc_binary} --version"` != "libprotoc ${protoc_version}"
       sh <<-EOF
         /bin/bash <<BASH
+
+        set -euo pipefail
         if [ ! -f #{protoc_binary} ] ; then
           echo "Downloading protoc #{protoc_version}"
           cd /tmp
@@ -45,72 +103,69 @@ BASH
           else
             curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v#{protoc_version}/protoc-#{protoc_version}-linux-x86_64.zip
           fi
-          unzip protoc-#{protoc_version}*.zip
+
+          mkdir -p #{toolchain_bin_dir}
+          rm -rf include/ # make sure there is no stale include
+          unzip -o protoc-#{protoc_version}*.zip
           mv bin/protoc #{protoc_binary}
-        fi
+
+          mkdir -p #{toolchain_include_dir}
+          rm -rf #{toolchain_include_dir}/**
+          mv include/* #{toolchain_include_dir}
+         fi
 BASH
-      EOF
-    end
-  end
+       EOF
+     end
+   end
+
+  task :protoc => ['install_protoc', 'setup_gogo'] do
 
-  task :protoc => ['install_protoc'] do
     sh <<-EOF
       /bin/bash <<BASH
       set -euo pipefail
 
       export GO111MODULE=auto
 
-      rm -rf #{gogo_dir}
-      rm -rf /tmp/gogo-bin-*
-
-      mkdir -p #{gogo_dir}/src/github.com/gogo
-      git clone https://github.com/gogo/protobuf.git #{gogo_dir}/src/github.com/gogo/protobuf
-
-      # Install v1.3.2
-      pushd #{gogo_dir}/src/github.com/gogo/protobuf
-      git checkout v1.3.2
-      GOBIN=/tmp/gogo-bin-v1.3.2 GOPATH=#{gogo_dir} make clean install
-
-      popd
-
       echo "Generating logs proto"
-      PATH=/tmp/gogo-bin-v1.3.2 #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto
+      PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto
 
       echo "Generating metrics proto (go)"
-      PATH=/tmp/gogo-bin-v1.3.2 #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src proto/metrics/agent_payload.proto
+      PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_include}:#{toolchain_legacy_include_dir}:. --gogofast_out=$GOPATH/src proto/metrics/agent_payload.proto
+      echo "done"
 
       echo "Generating metrics proto (python)"
-      #{protoc_legacy_binary} --proto_path=#{gogo_dir}/src:$GOPATH/src:./proto/metrics --python_out=python agent_payload.proto
+      PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=#{toolchain_legacy_include_dir}:#{gogo_include}:./proto/metrics --python_out=python agent_payload.proto
 
       echo "Generating process proto (go)"
-      PATH=/tmp/gogo-bin-v1.3.2 #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofaster_out=$GOPATH/src proto/process/*.proto
+      PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=#{toolchain_legacy_include_dir}:#{gogo_include}:. --gogofaster_out=$GOPATH/src proto/process/*.proto
 
-      GOPATH=#{protoc_gen_go_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:.  --gostreamer_out=$GOPATH/src proto/process/*.proto
+      GOPATH=#{toolchain_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
+      PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:.  --gostreamer_out=$GOPATH/src proto/process/*.proto
       mv v5/process/proto/process/*.go process
 
       # Install protoc-gen-go
-      GOPATH=#{protoc_gen_go_dir} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1
-      GOPATH=#{protoc_gen_go_dir} go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@#{protoc_jsonschema_version}
+      GOPATH=#{toolchain_dir} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1
+      GOPATH=#{toolchain_dir} go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@#{protoc_jsonschema_version}
+
 
       echo "Generating contlcycle proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/contlcycle/contlcycle.proto
+      PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/contlcycle/contlcycle.proto
 
       echo "Generating kubernetes autoscaling proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src --jsonschema_out=type_names_with_no_package:jsonschema proto/autoscaling/kubernetes/autoscaling.proto
+      PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=$GOPATH/src:#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src --jsonschema_out=type_names_with_no_package:jsonschema proto/autoscaling/kubernetes/autoscaling.proto
 
       echo "Generating contimage proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/contimage/contimage.proto
+      PATH=#{toolchain_bin_dir}  #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/contimage/contimage.proto
 
       echo "Generating sbom proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/deps/github.com/CycloneDX/specification/schema/bom-1.4.proto
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/sbom/sbom.proto
+      PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/deps/github.com/CycloneDX/specification/schema/bom-1.4.proto
+      PATH=#{toolchain_bin_dir}  #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/sbom/sbom.proto
 
       # Install protoc-gen-go-vtproto
-      GOPATH=#{protoc_gen_go_dir} go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.5.0
+      GOPATH=#{toolchain_dir} go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.5.0
 
       echo "Generating CWS Activity Dumps v1"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. \
+      PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=$GOPATH/src:. \
         --java_out=java \
         --go_out=$GOPATH/src \
         --go-vtproto_out=$GOPATH/src \