From c0ce5822d9748756c7734753f234a1c19464818d Mon Sep 17 00:00:00 2001 From: Casper Meijn Date: Sun, 10 Mar 2024 11:45:15 +0100 Subject: [PATCH] build: Use git submodule to download protobuf sources Instead of downloading a tarball, use a submodule to get the protobuf sources. Newer versions of protobuf require recursive submodules for its dependencies --- .github/workflows/ci.yml | 2 + .gitmodules | 3 ++ protobuf/Cargo.toml | 3 -- protobuf/build.rs | 96 +++++++++++++++++----------------------- third_party/protobuf | 1 + 5 files changed, 47 insertions(+), 58 deletions(-) create mode 100644 .gitmodules create mode 160000 third_party/protobuf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cef9d732..9ddcd346f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: - windows-latest steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - name: install toolchain (${{ matrix.toolchain }}) uses: dtolnay/rust-toolchain@master with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..a53f6e56f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/protobuf"] + path = third_party/protobuf + url = https://github.com/protocolbuffers/protobuf.git diff --git a/protobuf/Cargo.toml b/protobuf/Cargo.toml index d3131ce68..682f17285 100644 --- a/protobuf/Cargo.toml +++ b/protobuf/Cargo.toml @@ -14,10 +14,7 @@ prost-types = { path = "../prost-types" } [build-dependencies] anyhow = "1.0.1" -curl = "0.4.13" -flate2 = "1.0.3" prost-build = { path = "../prost-build" } -tar = "0.4.15" tempfile = "3" # With libz-sys `1.1.8` the msrv has been bumped to 1.54. Since, this dep diff --git a/protobuf/build.rs b/protobuf/build.rs index 51495ff47..6ab495be6 100644 --- a/protobuf/build.rs +++ b/protobuf/build.rs @@ -1,15 +1,9 @@ use std::env; use std::fs; -use std::io::Cursor; use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{ensure, Context, Result}; -use curl::easy::Easy; -use flate2::bufread::GzDecoder; -use tar::Archive; - -const VERSION: &str = "3.14.0"; static TEST_PROTOS: &[&str] = &[ "test_messages_proto2.proto", @@ -41,20 +35,29 @@ static DATASET_PROTOS: &[&str] = &[ fn main() -> Result<()> { let out_dir = &PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR environment variable not set")); - let protobuf_dir = &out_dir.join(format!("protobuf-{}", VERSION)); + + let src_dir = PathBuf::from("../third_party/protobuf"); + if !src_dir.join("cmake").exists() { + anyhow::bail!( + "protobuf sources are not checked out; Try `git submodule update --init --recursive`" + ) + } + + let version = git_describe(&src_dir)?; + let protobuf_dir = &out_dir.join(format!("protobuf-{}", version)); if !protobuf_dir.exists() { + apply_patches(&src_dir)?; let tempdir = tempfile::Builder::new() .prefix("protobuf") .tempdir_in(out_dir) .expect("failed to create temporary directory"); - let src_dir = &download_protobuf(tempdir.path())?; - let prefix_dir = &src_dir.join("prefix"); + let prefix_dir = &tempdir.path().join("prefix"); fs::create_dir(prefix_dir).expect("failed to create prefix directory"); - install_conformance_test_runner(src_dir, prefix_dir)?; - install_protos(src_dir, prefix_dir)?; - install_datasets(src_dir, prefix_dir)?; + install_conformance_test_runner(&src_dir, prefix_dir)?; + install_protos(&src_dir, prefix_dir)?; + install_datasets(&src_dir, prefix_dir)?; fs::rename(prefix_dir, protobuf_dir).context("failed to move protobuf dir")?; } @@ -100,44 +103,26 @@ fn main() -> Result<()> { Ok(()) } -fn download_tarball(url: &str, out_dir: &Path) -> Result<()> { - let mut data = Vec::new(); - let mut handle = Easy::new(); - - // Download the tarball. - handle.url(url).context("failed to configure tarball URL")?; - handle - .follow_location(true) - .context("failed to configure follow location")?; - { - let mut transfer = handle.transfer(); - transfer - .write_function(|new_data| { - data.extend_from_slice(new_data); - Ok(new_data.len()) - }) - .context("failed to write download data")?; - transfer.perform().context("failed to download tarball")?; +fn git_describe(src_dir: &Path) -> Result { + let output = Command::new("git") + .arg("describe") + .arg("--tags") + .arg("--always") + .current_dir(src_dir) + .output() + .context("Unable to describe protobuf git repo")?; + if !output.status.success() { + anyhow::bail!( + "Unable to describe protobuf git repo: {}", + String::from_utf8_lossy(&output.stderr) + ); } - - // Unpack the tarball. - Archive::new(GzDecoder::new(Cursor::new(data))) - .unpack(out_dir) - .context("failed to unpack tarball") + let stdout = String::from_utf8_lossy(&output.stdout); + Ok(stdout.trim().to_string()) } -/// Downloads and unpacks a Protobuf release tarball to the provided directory. -fn download_protobuf(out_dir: &Path) -> Result { - download_tarball( - &format!( - "https://github.com/google/protobuf/archive/v{}.tar.gz", - VERSION - ), - out_dir, - )?; - let src_dir = out_dir.join(format!("protobuf-{}", VERSION)); - - // Apply patches. +/// Apply patches to the protobuf source directory +fn apply_patches(src_dir: &Path) -> Result<()> { let mut patch_src = env::current_dir().context("failed to get current working directory")?; patch_src.push("src"); patch_src.push("fix-conformance_test_runner-cmake-build.patch"); @@ -149,9 +134,10 @@ fn download_protobuf(out_dir: &Path) -> Result { .current_dir(&src_dir) .status() .context("failed to apply patch")?; - ensure!(rc.success(), "protobuf patch failed"); + // exit code: 0 means success; 1 means already applied + ensure!(rc.code().unwrap() <= 1, "protobuf patch failed"); - Ok(src_dir) + Ok(()) } #[cfg(windows)] @@ -188,7 +174,7 @@ fn install_conformance_test_runner(src_dir: &Path, prefix_dir: &Path) -> Result< ensure!(rc.success(), "failed to make protobuf"); // Install the conformance-test-runner binary, since it isn't done automatically. - fs::rename( + fs::copy( src_dir.join("conformance_test_runner"), prefix_dir.join("bin").join("conformance-test-runner"), ) @@ -204,7 +190,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> { let test_include_dir = &include_dir.join("google").join("protobuf"); fs::create_dir_all(test_include_dir).expect("failed to create test include directory"); for proto in TEST_PROTOS { - fs::rename( + fs::copy( src_dir .join("src") .join("google") @@ -219,7 +205,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> { let conformance_include_dir = &include_dir.join("conformance"); fs::create_dir(conformance_include_dir) .expect("failed to create conformance include directory"); - fs::rename( + fs::copy( src_dir.join("conformance").join("conformance.proto"), conformance_include_dir.join("conformance.proto"), ) @@ -231,7 +217,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> { let datasets_src_dir = &benchmarks_src_dir.join("datasets"); let datasets_include_dir = &benchmarks_include_dir.join("datasets"); fs::create_dir(benchmarks_include_dir).expect("failed to create benchmarks include directory"); - fs::rename( + fs::copy( benchmarks_src_dir.join("benchmarks.proto"), benchmarks_include_dir.join("benchmarks.proto"), ) @@ -240,7 +226,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> { let dir = &datasets_include_dir.join(proto.parent().unwrap()); fs::create_dir_all(dir) .with_context(|| format!("unable to create directory {}", dir.display()))?; - fs::rename( + fs::copy( datasets_src_dir.join(proto), datasets_include_dir.join(proto), ) @@ -262,7 +248,7 @@ fn install_datasets(src_dir: &Path, prefix_dir: &Path) -> Result<()> { .join("dataset.google_message1_proto3.pb"), Path::new("google_message2").join("dataset.google_message2.pb"), ] { - fs::rename( + fs::copy( src_dir.join("benchmarks").join("datasets").join(dataset), share_dir.join(dataset.file_name().unwrap()), ) diff --git a/third_party/protobuf b/third_party/protobuf new file mode 160000 index 000000000..2514f0bd7 --- /dev/null +++ b/third_party/protobuf @@ -0,0 +1 @@ +Subproject commit 2514f0bd7da7e2af1bed4c5d1b84f031c4d12c10