Skip to content

Commit

Permalink
Download all dependencies (protoc, gogo)
Browse files Browse the repository at this point in the history 
Adds a `rake setup_gogo` task which will clone gogo into a new toolchains directory (which is git ignored).

On every run of codegen, we'll checkout a pinned version of gogo and rebuild it. This should be pretty fast as long as gogo is already checked out.

This PR also adds a new 'toolchains' location where all dependencies are cached. Before we were jumbling a bunch of dependencies into /tmp.

This led to a weird case where we were accidentally (?) using a new protobuf compiler with a legacy .proto library (for stuff like google.Duration). This is now explicit.

The structure of the new toolchains is described in the Rakefile
  • Loading branch information
@leeavital
leeavital committed Sep 3, 2024
1 parent 00b3937 commit b2c1e8e
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 46 deletions.
6 changes: 5 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,8 @@
# jetbrains IDE directory
/.idea/

vendor/
vendor/

# build time dependencies that are either downloaded or
# built during codegen
toolchains/
147 changes: 102 additions & 45 deletions Rakefile
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,70 @@
#
# Rakefile for agent-payload
#

# where all toolchains (protobuf compilers, protobuf plugins,etc) are stored
toolchain_dir=File.join(File.dirname(__FILE__ ), "toolchains")

# binaries
toolchain_bin_dir=File.join(toolchain_dir, "bin")

# include path for the legacy and main protobuf compilers respectivicey
toolchain_include_dir=File.join(toolchain_dir, "/include/proto")
toolchain_legacy_include_dir=File.join(toolchain_dir, "/include/proto_legacy")

protoc_legacy_version="3.5.1"
protoc_legacy_binary="/tmp/protoc#{protoc_legacy_version}"
protoc_legacy_binary=File.join(toolchain_bin_dir, "/protoc" + protoc_legacy_version)

protoc_version="21.12"
protoc_binary="/tmp/protoc#{protoc_version}"
protoc_binary=File.join(toolchain_bin_dir, "/protoc" + protoc_version)

gogo_dir="/tmp/gogo"
protoc_gen_go_dir="/tmp/protoc-gen-go"
gogo_tag = "v1.3.2"
gogo_dir=File.join(toolchain_dir, "/gogo")
gogo_include = "#{toolchain_dir}/gogo/src:#{toolchain_dir}/gogo/src/github.com/gogo/protobuf/protobuf"
gogo_bin = File.join(toolchain_bin_dir, "/gogo-bin-#{gogo_tag}")
protoc_jsonschema_version="73d5723"

### toolchains is meant to store a cache of all binary dependencies needed to build the agent-payload.
### this rakefile will download those dependencies on the fly if needed.
### the structure of toolchains is as follows:
### /toolchains
### /toolchains/bin -- contains any binaries used for building
### /toolchains/bin/protoc21.12 -- the new protobuf compiler
### /toolchains/bin/protoc3.5.1 -- the legacy protobuf compiler
### /toolchains/bin/protoc-gen-go -- the go code generator for protoc
### /toolchains/bin/protoc-gen-XXX -- other protobuf generators (vtproto, java, etc.)
### /toolchains/include -- contains any protobuf library files for common types (like proto.Duration)
### /toolchains/includes/proto -- protobuf libraries bundled with the new protobuf compiler
### /toolchains/includes/proto_legacy -- protobuf libraries bundled with the legacy protobuf compiler
### => /toolchains/gogo -- temp directory used to build the gogo_faster generator

namespace :codegen do
task :clean do
sh "rm -rf #{gogo_dir}"
sh "rm -rf #{toolchain_dir}"
end

task :setup_gogo => ['install_protoc_all'] do
if ! Dir.exist?(gogo_dir) then
directory "#{gogo_dir}/src/github.com/gogo"
sh "git clone https://github.com/gogo/protobuf.git #{gogo_dir}/src/github.com/gogo/protobuf"
else
puts "gogo already cloned into #{gogo_dir}"
end

Dir.chdir("#{gogo_dir}/src/github.com/gogo/protobuf") do
sh "git checkout #{gogo_tag}"
sh "PATH=$PATH:/tmp GOBIN=#{toolchain_bin_dir} GOPATH=#{gogo_dir} make clean install"
end
end


task 'install_protoc_all' => ['install_protoc', 'install_protoc_legacy']

task :install_protoc do
if `bash -c "protoc --version"` != "libprotoc ${protoc_legacy_version}"
task :install_protoc_legacy do
if `bash -c "#{protoc_legacy_binary} --version"` != "libprotoc ${protoc_version}"
sh <<-EOF
/bin/bash <<BASH
set -euo pipefail
if [ ! -f #{protoc_legacy_binary} ] ; then
echo "Downloading protoc #{protoc_legacy_version}"
cd /tmp
Expand All @@ -27,15 +73,26 @@ namespace :codegen do
else
curl -OL https://github.com/google/protobuf/releases/download/v#{protoc_legacy_version}/protoc-#{protoc_legacy_version}-linux-x86_64.zip
fi
unzip protoc-#{protoc_legacy_version}*.zip
mkdir -p #{toolchain_bin_dir}
unzip -o protoc-#{protoc_legacy_version}*.zip
mv bin/protoc #{protoc_legacy_binary}
mkdir -p #{toolchain_legacy_include_dir}
rm -rf #{toolchain_legacy_include_dir}/**
mv include/* #{toolchain_legacy_include_dir}/
fi
BASH
EOF
end
if `bash -c "protoc --version"` != "libprotoc ${protoc_version}"
end


task :install_protoc do
if `bash -c "#{protoc_binary} --version"` != "libprotoc ${protoc_version}"
sh <<-EOF
/bin/bash <<BASH
set -euo pipefail
if [ ! -f #{protoc_binary} ] ; then
echo "Downloading protoc #{protoc_version}"
cd /tmp
Expand All @@ -44,72 +101,72 @@ BASH
else
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v#{protoc_version}/protoc-#{protoc_version}-linux-x86_64.zip
fi
unzip protoc-#{protoc_version}*.zip
mkdir -p #{toolchain_bin_dir}
unzip -o protoc-#{protoc_version}*.zip
mv bin/protoc #{protoc_binary}
fi
BASH
EOF
end
end
task :protoc => ['install_protoc'] do
mkdir -p #{toolchain_include_dir}
echo "here"
rm -rf #{toolchain_include_dir}/**
mv include/* #{toolchain_include_dir}
fi
BASH
EOF
end
end

task :protoc => ['install_protoc', 'setup_gogo'] do

puts "Generating logs proto"
sh "PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto"
sh <<-EOF
/bin/bash <<BASH
set -euo pipefail
export GO111MODULE=auto
rm -rf #{gogo_dir}
rm -rf /tmp/gogo-bin-*
mkdir -p #{gogo_dir}/src/github.com/gogo
git clone https://github.com/gogo/protobuf.git #{gogo_dir}/src/github.com/gogo/protobuf
# Install v1.3.2
pushd #{gogo_dir}/src/github.com/gogo/protobuf
git checkout v1.3.2
GOBIN=/tmp/gogo-bin-v1.3.2 GOPATH=#{gogo_dir} make clean install
popd
echo "Generating logs proto"
PATH=/tmp/gogo-bin-v1.3.2 #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto
PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src --java_out=java proto/logs/agent_logs_payload.proto
echo "Generating metrics proto (go)"
PATH=/tmp/gogo-bin-v1.3.2 #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofast_out=$GOPATH/src proto/metrics/agent_payload.proto
PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_include}:#{toolchain_legacy_include_dir}:. --gogofast_out=$GOPATH/src proto/metrics/agent_payload.proto
echo "done"
echo "Generating metrics proto (python)"
#{protoc_legacy_binary} --proto_path=#{gogo_dir}/src:$GOPATH/src:./proto/metrics --python_out=python agent_payload.proto
PATH=#{toolchain_bin_dir} #{protoc_legacy_binary} --proto_path=#{toolchain_legacy_include_dir}:#{gogo_include}:./proto/metrics --python_out=python agent_payload.proto
echo "Generating process proto (go)"
PATH=/tmp/gogo-bin-v1.3.2 #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gogofaster_out=$GOPATH/src proto/process/*.proto
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:#{gogo_include}:. --gogofaster_out=$GOPATH/src proto/process/*.proto
GOPATH=#{protoc_gen_go_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
PATH=#{protoc_gen_go_dir}/bin #{protoc_legacy_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gostreamer_out=$GOPATH/src proto/process/*.proto
GOPATH=#{toolchain_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
GOPATH=#{toolchain_dir} go install github.com/leeavital/protoc-gen-gostreamer@v0.1.0
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=$GOPATH/src:#{gogo_dir}/src:. --gostreamer_out=$GOPATH/src proto/process/*.proto
mv v5/process/proto/process/*.go process
# Install protoc-gen-go
GOPATH=#{protoc_gen_go_dir} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1
GOPATH=#{protoc_gen_go_dir} go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@#{protoc_jsonschema_version}
GOPATH=#{toolchain_dir} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.1
GOPATH=#{toolchain_dir} go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@#{protoc_jsonschema_version}
echo "Generating contlcycle proto"
PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/contlcycle/contlcycle.proto
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/contlcycle/contlcycle.proto
echo "Generating kubernetes autoscaling proto"
PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src --jsonschema_out=type_names_with_no_package:jsonschema proto/autoscaling/kubernetes/autoscaling.proto
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=$GOPATH/src:#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src --jsonschema_out=type_names_with_no_package:jsonschema proto/autoscaling/kubernetes/autoscaling.proto
echo "Generating contimage proto"
PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/contimage/contimage.proto
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/contimage/contimage.proto
echo "Generating sbom proto"
PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/deps/github.com/CycloneDX/specification/schema/bom-1.4.proto
PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. --go_out=$GOPATH/src proto/sbom/sbom.proto
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/deps/github.com/CycloneDX/specification/schema/bom-1.4.proto
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=#{toolchain_legacy_include_dir}:. --go_out=$GOPATH/src proto/sbom/sbom.proto
# Install protoc-gen-go-vtproto
GOPATH=#{protoc_gen_go_dir} go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.5.0
GOPATH=#{toolchain_dir} go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.5.0
echo "Generating CWS Activity Dumps v1"
PATH=#{protoc_gen_go_dir}/bin #{protoc_binary} --proto_path=$GOPATH/src:. \
PATH=#{toolchain_bin_dir} #{protoc_binary} --proto_path=$GOPATH/src:. \
--java_out=java \
--go_out=$GOPATH/src \
--go-vtproto_out=$GOPATH/src \
Expand Down

0 comments on commit b2c1e8e

Please sign in to comment.