Avoid re-downloading and rebuilding gogo every time we run codegen

Adds a `rake setup_gogo` task which will clone gogo into a new toolchains directory (which is git ignored). On every run of codegen, we'll checkout a pinned version of gogo and rebuild it. This should be pretty fast as long as gogo is already checked out.
DataDog · Aug 31, 2024 · cb1c8de · cb1c8de
1 parent 22f3ea6
commit cb1c8de
Show file tree

Hide file tree

Showing 11 changed files with 477 additions and 1,136 deletions.
diff --git a/.Rakefile.swo b/.Rakefile.swo
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -41,7 +41,7 @@ jobs:
       uses: actions/setup-go@v3
       with:
         go-version-file: "/home/runner/work/agent-payload/agent-payload/go/src/github.com/DataDog/agent-payload/go.mod"
-    - run: rake codegen
+    - run: rake codegen --trace
       env:
         GOPATH: "/home/runner/work/agent-payload/agent-payload/go"
     - name: Check for diffs

diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,8 @@
 # jetbrains IDE directory
 /.idea/
 
-vendor/
+vendor/
+
+# build time dependencies that are either downloaded or
+# built during codegen
+toolchains/
diff --git a/Rakefile b/Rakefile
@@ -1,19 +1,45 @@
-#
 # Rakefile for agent-payload
 #
 
 protoc_binary="protoc"
 protoc_version="3.5.1"
-gogo_dir="/tmp/gogo"
+toolchain_dir=Dir.pwd + "/toolchains"
+toolchain_bin_dir=toolchain_dir + "/bin"
+toolchain_include2_dir=toolchain_dir + "/include/proto2"
+toolchain_include_dir=toolchain_dir + "/include/proto"
+
+gogo_dir=toolchain_dir+ "/gogo"
+gogo_include = "#{toolchain_dir}/gogo/src:#{toolchain_dir}/gogo/src/github.com/gogo/protobuf/protobuf"
+gogo_bin = toolchain_bin_dir + "/gogo-bin-v1.3.2"
 protoc_binary_2="protoc"
-protoc_version_2="21.12"
-protoc_gen_go_dir="/tmp/protoc-gen-go"
+protoc_version_2="22.5"
 protoc_jsonschema_version="73d5723"
 
 
 namespace :codegen do
+  task :clean do
+      sh "rm -rf #{gogo_dir}"
+      sh "rm -rf #{toolchain_dir}"
+  end
+
+  task :setup_gogo => ['install_protoc'] do
+    if ! Dir.exist?(gogo_dir) then
+      sh "mkdir -p #{gogo_dir}/src/github.com/gogo"
+      sh "git clone https://github.com/gogo/protobuf.git #{gogo_dir}/src/github.com/gogo/protobuf"
+    else
+      puts "gogo already cloned into #{gogo_dir}"
+    end
+
+    Dir.chdir("#{gogo_dir}/src/github.com/gogo/protobuf") do
+      sh "git checkout v1.3.2"
+      sh "PATH=$PATH:/tmp  GOBIN=#{toolchain_bin_dir} GOPATH=#{gogo_dir} make clean install"
+    end
+  end
+
 
-  task :install_protoc do
+  task 'install_protoc' => ['install_protoc_2', 'install_protoc_3']
+
+  task :install_protoc_3 => ['codegen:install_protoc_2'] do
     if `bash -c "protoc --version"` != "libprotoc ${protoc_version}"
       protoc_binary="/tmp/protoc#{protoc_version}"
       sh <<-EOF
@@ -26,12 +52,22 @@ namespace :codegen do
           else
             curl -OL https://github.com/google/protobuf/releases/download/v#{protoc_version}/protoc-#{protoc_version}-linux-x86_64.zip
           fi
-          unzip protoc-#{protoc_version}*.zip
-          mv bin/protoc #{protoc_binary}
+          unzip -o protoc-#{protoc_version}*.zip
+          mkir -p #{toolchain_bin_dir}
+          mv bin/protoc #{toolchain_bin_dir}/
+
+
+          mkdir -p #{toolchain_include_dir}
+          rm -rf #{toolchain_include_dir}/**
+          mv include/* #{toolchain_include_dir}/
         fi
 BASH
       EOF
     end
+  end
+
+
+  task :install_protoc_2 do
     if `bash -c "protoc --version"` != "libprotoc ${protoc_version_2}"
       protoc_binary_2="/tmp/protoc#{protoc_version_2}"
       sh <<-EOF
@@ -44,72 +80,70 @@ BASH
           else
             curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v#{protoc_version_2}/protoc-#{protoc_version_2}-linux-x86_64.zip
           fi
-          unzip protoc-#{protoc_version_2}*.zip
-          mv bin/protoc #{protoc_binary_2}
+          unzip -o protoc-#{protoc_version_2}*.zip
+          mkdir -p #{toolchain_bin_dir}
+          mv bin/protoc #{toolchain_bin_dir}/protoc2
+
+          mkdir -p #{toolchain_include2_dir}
+          echo "here"
+          rm -rf #{toolchain_include2_dir}/**
+          mv include/* #{toolchain_include2_dir}
         fi
 BASH
       EOF
-    end
   end
+  end
+
+  task :protoc => ['install_protoc', 'setup_gogo'] do
 
-  task :protoc => ['install_protoc'] do
+    puts "Generating logs proto"
+    sh "PATH=#{toolchain_bin_dir} protoc --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto"
     sh <<-EOF
       /bin/bash <<BASH
       set -euo pipefail
 
       export GO111MODULE=auto
 
-      rm -rf #{gogo_dir}
-      rm -rf /tmp/gogo-bin-*
-
-      mkdir -p #{gogo_dir}/src/github.com/gogo
-      git clone https://github.com/gogo/protobuf.git #{gogo_dir}/src/github.com/gogo/protobuf
-
-      # Install v1.3.2
-      pushd #{gogo_dir}/src/github.com/gogo/protobuf
-      git checkout v1.3.2
-      GOBIN=/tmp/gogo-bin-v1.3.2 GOPATH=#{gogo_dir} make clean install
-
-      popd
-
       echo "Generating logs proto"
-      PATH=/tmp/gogo-bin-v1.3.2 #{protoc_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto
+      PATH=#{toolchain_bin_dir} protoc --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto
 
       echo "Generating metrics proto (go)"
-      PATH=/tmp/gogo-bin-v1.3.2 #{protoc_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src proto/metrics/agent_payload.proto
+      PATH=#{toolchain_bin_dir} protoc --proto_path=$GOPATH/src:#{gogo_include}:#{toolchain_include_dir}:. --gogofast_out=$GOPATH/src proto/metrics/agent_payload.proto
+      echo "done"
 
       echo "Generating metrics proto (python)"
-      #{protoc_binary} --proto_path=#{gogo_dir}/src:$GOPATH/src:./proto/metrics --python_out=python agent_payload.proto
+      PATH=#{toolchain_bin_dir} protoc --proto_path=#{toolchain_include_dir}:#{gogo_include}:./proto/metrics --python_out=python agent_payload.proto
 
       echo "Generating process proto (go)"
-      PATH=/tmp/gogo-bin-v1.3.2 #{protoc_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofaster_out=$GOPATH/src proto/process/*.proto
+      PATH=#{toolchain_bin_dir} protoc --proto_path=#{toolchain_include_dir}:#{gogo_include}:. --gogofaster_out=$GOPATH/src proto/process/*.proto
 
-      GOPATH=#{protoc_gen_go_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:.  --gostreamer_out=$GOPATH/src proto/process/*.proto
+      GOPATH=#{toolchain_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
+      PATH=#{toolchain_bin_dir} protoc --proto_path=$GOPATH/src:#{gogo_dir}/src:.  --gostreamer_out=$GOPATH/src proto/process/*.proto
       mv v5/process/proto/process/*.go process
 
       # Install protoc-gen-go
-      GOPATH=#{protoc_gen_go_dir} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1
-      GOPATH=#{protoc_gen_go_dir} go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@#{protoc_jsonschema_version}
+      GOPATH=#{toolchain_dir} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1
+      GOPATH=#{toolchain_dir} go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@#{protoc_jsonschema_version}
+
 
       echo "Generating contlcycle proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary_2} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/contlcycle/contlcycle.proto
+      PATH=#{toolchain_bin_dir} protoc2 --proto_path=#{toolchain_include_dir}:. --go_out=$GOPATH/src proto/contlcycle/contlcycle.proto
 
       echo "Generating kubernetes autoscaling proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary_2} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src --jsonschema_out=type_names_with_no_package:jsonschema proto/autoscaling/kubernetes/autoscaling.proto
+      PATH=#{toolchain_bin_dir} protoc2 --proto_path=$GOPATH/src:#{toolchain_include2_dir}:. --go_out=$GOPATH/src --jsonschema_out=type_names_with_no_package:jsonschema proto/autoscaling/kubernetes/autoscaling.proto
 
       echo "Generating contimage proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary_2} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/contimage/contimage.proto
+      PATH=#{toolchain_bin_dir}  protoc2 --proto_path=#{toolchain_include2_dir}:. --go_out=$GOPATH/src proto/contimage/contimage.proto
 
       echo "Generating sbom proto"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary_2} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/deps/github.com/CycloneDX/specification/schema/bom-1.4.proto
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary_2} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/sbom/sbom.proto
+      PATH=#{toolchain_bin_dir} protoc2 --proto_path=#{toolchain_include2_dir}:. --go_out=$GOPATH/src proto/deps/github.com/CycloneDX/specification/schema/bom-1.4.proto
+      PATH=#{toolchain_bin_dir}  protoc2 --proto_path=#{toolchain_include2_dir}:. --go_out=$GOPATH/src proto/sbom/sbom.proto
 
       # Install protoc-gen-go-vtproto
-      GOPATH=#{protoc_gen_go_dir} go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.5.0
+      GOPATH=#{toolchain_dir} go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.5.0
 
       echo "Generating CWS Activity Dumps v1"
-      PATH=#{protoc_gen_go_dir}/bin #{protoc_binary_2} --proto_path=$GOPATH/src:. \
+      PATH=#{toolchain_bin_dir} protoc2 --proto_path=$GOPATH/src:. \
         --java_out=java \
         --go_out=$GOPATH/src \
         --go-vtproto_out=$GOPATH/src \

diff --git a/autoscaling/kubernetes/autoscaling.pb.go b/autoscaling/kubernetes/autoscaling.pb.go
diff --git a/contimage/contimage.pb.go b/contimage/contimage.pb.go
diff --git a/contlcycle/contlcycle.pb.go b/contlcycle/contlcycle.pb.go
diff --git a/cws/dumpsv1/activity_dump.pb.go b/cws/dumpsv1/activity_dump.pb.go