Release 0.14.0 (2023-11-07)

### Features * [API breaking change] Rename subkey to hostname. * [Sharding] Add a tool to validate records * [Sharding] Generate shard specific delta files * Add base64 encoding to CSV delta cli tool * Add safe metric definitions * Add unsafe metric definitions * Fully statically link the server binary * Simplified UDF tester. ### Bug Fixes * Add error message when key not found. * Add proxy visibility back to aws_artifacts * allow internal ingress for otlp * Clear errors in driver prior to parsing. * Fix bash script flag for -v * Fully qualify RegisterBenchmark calls. * Package proxify layer on container_image * Remove unused import httpbody.proto * Update visibility target to public target in common repo * Use set for cache key lookups ### GCP: Features * **GCP:** add capability to use existing service_mesh ### Documentation * Add a top level getting_started directory. * Add docs about AWS AMI structure * Add instructions on how to call UDF APIs from C++ WASM. * Update sharding docs with GCP-specific info ### Dependencies * **deps:** Update data-plane-shared to b463f16 * **deps:** Utilize rules_closure deps via data-plane-shared Bug: N/A Change-Id: I131da99f79b6f24613768cfc50485946678b3ee4 GitOrigin-RevId: 3ffac8eae058fb89585fe99d40243b0e17f3c352
privacysandbox · Nov 8, 2023 · 186e758 · 186e758
1 parent 4f18201
commit 186e758
Show file tree

Hide file tree

Showing 399 changed files with 5,437 additions and 4,186 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -1,2 +1,3 @@
 bazel-kv-server
 tools/wasm_example/
+google_internal/
diff --git a/.hadolint.yaml b/.hadolint.yaml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,9 +16,9 @@
 # See https://pre-commit.com/hooks.html for more hooks
 exclude: (?x)^(
              bazel-(bin|out|testlogs|workspace)/.*|
-             google_internal/coordinator_environments/.*$|
              .bazel_output/.*|
              builders/.*|
+             .*/BUILD|
              testing/functionaltest-system/.*|
              version.txt
          )$

diff --git a/BUILD → BUILD.bazel b/BUILD → BUILD.bazel
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,53 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## 0.14.0 (2023-11-07)
+
+
+### Features
+
+* [API breaking change] Rename subkey to hostname.
+* [Sharding] Add a tool to validate records
+* [Sharding] Generate shard specific delta files
+* Add base64 encoding to CSV delta cli tool
+* Add safe metric definitions
+* Add unsafe metric definitions
+* Fully statically link the server binary
+* Simplified UDF tester.
+
+
+### Bug Fixes
+
+* Add error message when key not found.
+* Add proxy visibility back to aws_artifacts
+* allow internal ingress for otlp
+* Clear errors in driver prior to parsing.
+* Fix bash script flag for -v
+* Fully qualify RegisterBenchmark calls.
+* Package proxify layer on container_image
+* Remove unused import httpbody.proto
+* Update visibility target to public target in common repo
+* Use set for cache key lookups
+
+
+### GCP: Features
+
+* **GCP:** add capability to use existing service_mesh
+
+
+### Documentation
+
+* Add a top level getting_started directory.
+* Add docs about AWS AMI structure
+* Add instructions on how to call UDF APIs from C++ WASM.
+* Update sharding docs with GCP-specific info
+
+
+### Dependencies
+
+* **deps:** Update data-plane-shared to b463f16
+* **deps:** Utilize rules_closure deps via data-plane-shared
+
 ## 0.13.0 (2023-10-10)
 
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -13,11 +13,11 @@ python_deps("//builders/bazel")
 
 http_archive(
     name = "google_privacysandbox_servers_common",
-    # commit 09042cd502156e341626d0e94560b2cac4e322a2 2023-10-09
-    sha256 = "12fa1c6d322a3b5034e12a78275fc02b535bcbaefd70426230ad675b84906472",
-    strip_prefix = "data-plane-shared-libraries-09042cd502156e341626d0e94560b2cac4e322a2",
+    # commit 3a47a5e 2023-11-01
+    sha256 = "d1a5cc31686e04f7c192f103600431f804689fbea54722250484ef38f9c8c7bf",
+    strip_prefix = "data-plane-shared-libraries-bd2efade9f130c7c8e0dbe964183204c6ee220ce",
     urls = [
-        "https://github.com/privacysandbox/data-plane-shared-libraries/archive/09042cd502156e341626d0e94560b2cac4e322a2.tar.gz",
+        "https://github.com/privacysandbox/data-plane-shared-libraries/archive/bd2efade9f130c7c8e0dbe964183204c6ee220ce.zip",
     ],
 )
 
@@ -45,7 +45,7 @@ load("@google_privacysandbox_servers_common//third_party:deps4.bzl", data_plane_
 data_plane_shared_deps4()
 
 load(
-    "//third_party:cpp_repositories.bzl",
+    "//third_party_deps:cpp_repositories.bzl",
     "cpp_repositories",
     "emscripten_repositories",
 )
@@ -54,15 +54,15 @@ cpp_repositories()
 
 EMSCRIPTEN_VER = emscripten_repositories()
 
-load("//third_party:container_deps.bzl", "container_deps")
+load("//third_party_deps:container_deps.bzl", "container_deps")
 
 container_deps()
 
-load("//third_party:emscripten_deps1.bzl", "emscripten_deps1")
+load("//third_party_deps:emscripten_deps1.bzl", "emscripten_deps1")
 
 emscripten_deps1()
 
-load("//third_party:emscripten_deps2.bzl", "emscripten_deps2")
+load("//third_party_deps:emscripten_deps2.bzl", "emscripten_deps2")
 
 emscripten_deps2(EMSCRIPTEN_VER)
 
@@ -112,7 +112,7 @@ load("@rules_flex//flex:flex.bzl", "flex_register_toolchains")
 
 flex_register_toolchains(version = "2.6.4")
 
-load("//third_party:python_deps.bzl", "python_repositories")
+load("//third_party_deps:python_deps.bzl", "python_repositories")
 
 python_repositories()
 
@@ -122,10 +122,6 @@ load("@word2vec//:requirements.bzl", "install_deps")
 # Call it to define repos for your requirements.
 install_deps()
 
-load("//third_party:rules_closure_repositories.bzl", "rules_closure_repositories")
+load("//third_party_deps:rules_closure_repositories.bzl", "rules_closure_repositories")
 
 rules_closure_repositories()
-
-load("//third_party:rules_closure_deps.bzl", "rules_closure_deps")
-
-rules_closure_deps()
diff --git a/builders/CHANGELOG.md b/builders/CHANGELOG.md
@@ -2,6 +2,55 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## 0.48.0 (2023-10-11)
+
+
+### Features
+
+* Add tools/bazel-amazonlinux2023
+
+
+### Bug Fixes
+
+* Add lldb symlink
+
+## 0.47.0 (2023-10-05)
+
+
+### Features
+
+* Add --cmd-profiler flag to tools/cbuild
+* Add google-pprof to coverage-tools image
+* Add libprofiler.so to build-debian image
+
+
+### Bug Fixes
+
+* Indicate default image in help text
+
+## 0.46.1 (2023-10-04)
+
+
+### Bug Fixes
+
+* cbuild exits normally even if bazel symlinks aren't normalized
+* Revert to clang 15.x
+* Use bash when normalize-dist runs inside docker
+* Use DOCKER_NETWORK if already set
+
+## 0.46.0 (2023-09-22)
+
+
+### Bug Fixes
+
+* Hide normalize-bazel-symlinks in cbuild
+
+
+### Dependencies
+
+* **deps:** Update amazonlinux images to Aug 2023
+* **deps:** Upgrade clang to v16 in build-debian
+
 ## 0.45.0 (2023-09-19)
 
 

diff --git a/builders/images/build-amazonlinux2/Dockerfile b/builders/images/build-amazonlinux2/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM amazonlinux:2.0.20230719.0
+FROM amazonlinux:2.0.20230822.0
 
 COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/
 COPY get_workspace_mount /usr/local/bin

diff --git a/builders/images/build-amazonlinux2023/Dockerfile b/builders/images/build-amazonlinux2023/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM amazonlinux:2023.1.20230719.0
+FROM amazonlinux:2023.1.20230825.0
 
 COPY /install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/
 COPY get_workspace_mount /usr/local/bin

diff --git a/builders/images/build-debian/Dockerfile b/builders/images/build-debian/Dockerfile
@@ -12,16 +12,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ARG BASE_IMAGE=ubuntu:20.04
+
+# ignore this hadolint error as BASE_IMAGE contains an image tag
+# hadolint ignore=DL3006
+FROM ${BASE_IMAGE} as libprofiler-builder
+ENV CC=clang \
+    CXX=clang
+ADD https://github.com/gperftools/gperftools/releases/download/gperftools-2.13/gperftools-2.13.tar.gz /build/gperftools.tar.gz
+ADD https://apt.llvm.org/llvm.sh /build/llvm.sh
+COPY compile_libprofiler /scripts/
+RUN /scripts/compile_libprofiler
+
 FROM docker/buildx-bin:v0.10 AS buildx-bin
-FROM ubuntu:20.04
 
+# ignore this hadolint error as BASE_IMAGE contains an image tag
+# hadolint ignore=DL3006
+FROM ${BASE_IMAGE}
+ARG LOCALE=en_US.UTF-8
+ARG TARGETARCH
 COPY install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/
 COPY get_workspace_mount /usr/local/bin
 COPY gitconfig /etc
 COPY --from=buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx
+COPY --from=libprofiler-builder /usr/lib/libprofiler.so.* /usr/lib/x86_64-linux-gnu/
+RUN \
+  ln -rs /usr/lib/x86_64-linux-gnu/libprofiler.so.0.* /usr/lib/x86_64-linux-gnu/libprofiler.so.0 && \
+  ln -rs /usr/lib/x86_64-linux-gnu/libprofiler.so.0.* /usr/lib/x86_64-linux-gnu/libprofiler.so
 
-ARG LOCALE=en_US.UTF-8
-ARG TARGETARCH
 ENV BUILD_ARCH="${TARGETARCH}" \
     WORKSPACE=/src/workspace \
     CC=clang \

diff --git a/builders/images/build-debian/compile_libprofiler b/builders/images/build-debian/compile_libprofiler
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -o pipefail
+set -o errexit
+
+declare -r -i CLANG_VER=15
+
+function apt_update() {
+  apt-get --quiet -o 'Acquire::https::No-Cache=True' -o 'Acquire::http::No-Cache=True' update
+}
+
+function install_build_tools() {
+  DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y --no-install-recommends \
+    gnupg="2.2.*" \
+    lsb-release="11.1.*" \
+    make="4.*" \
+    software-properties-common="0.99.*" \
+    wget="1.20.*"
+}
+
+function install_clang() {
+  ls -l /build
+  chmod +x /build/llvm.sh
+  /build/llvm.sh ${CLANG_VER}
+  apt-get --quiet install -y --no-install-recommends libc++-${CLANG_VER}-dev
+  update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VER} 100
+  rm -f llvm.sh
+
+  clang --version
+}
+
+function install_profiler() {
+  cd /build
+  tar xz --strip-components 1 -f gperftools.tar.gz
+  ./configure
+  make libprofiler.la
+  declare -r ver=0.5.9
+  cp .libs/libprofiler.so.${ver} /usr/lib
+  ls -l /usr/lib/libprofiler*
+}
+
+apt_update
+install_build_tools
+install_clang
+install_profiler
diff --git a/builders/images/build-debian/install_apps b/builders/images/build-debian/install_apps
@@ -95,7 +95,7 @@ function install_clang() {
   chmod +x llvm.sh
   ./llvm.sh ${CLANG_VER}
   apt-get --quiet install -y --no-install-recommends libc++-${CLANG_VER}-dev
-  for prog in clang lld ld.lld llvm-cov llvm-profdata; do
+  for prog in clang lldb lld ld.lld llvm-cov llvm-profdata; do
     update-alternatives --install /usr/bin/${prog} ${prog} /usr/bin/${prog}-${CLANG_VER} 100
   done
   rm -f llvm.sh

diff --git a/builders/images/coverage-tools/install_apps b/builders/images/coverage-tools/install_apps
@@ -33,7 +33,8 @@ function apt_update() {
 
 function install_misc() {
   DEBIAN_FRONTEND=noninteractive apt-get --quiet install -y --no-install-recommends \
-    lcov="1.*"
+    lcov="1.*" \
+    google-perftools="2.*"
 }
 
 function clean_debian() {

diff --git a/builders/tests/data/hashes/build-amazonlinux2 b/builders/tests/data/hashes/build-amazonlinux2
@@ -1 +1 @@
-7ecd31d8ca27e17acf475eee1fdf719e60310bf63920974294f2ad8bf87f014c
+1feb61b0cf40e2797fc6a3425c3e50ce928ca194f3338d9a7155e51a4917312a
diff --git a/builders/tests/data/hashes/build-amazonlinux2023 b/builders/tests/data/hashes/build-amazonlinux2023
@@ -1 +1 @@
-d8715c569f95b1162afcfbf7892d9d87815063be8b84475e7605f712ffca7461
+a1c165d019f546e1fcad06506da3b274094b61a9add5576bf3fed6e5a45fb8d4
diff --git a/builders/tests/data/hashes/build-debian b/builders/tests/data/hashes/build-debian
@@ -1 +1 @@
-f4c5ca3eb72c967ec7f2ba8cd4236fc8c7af9b211b99b8a32c3a9656ca04ba29
+a10495fe1e23472aea6770b0c9760e1cdfc011a4883ff5fb3d488774e6995ae6
diff --git a/builders/tests/data/hashes/coverage-tools b/builders/tests/data/hashes/coverage-tools
@@ -1 +1 @@
-4b909561789591bd680993312cf708e831da57ffe9d8e764e03b190d30073ce0
+cd3fb189dd23793af3bdfa02d6774ccb35bddbec7059761e25c4f7be4c1e8ca1
diff --git a/builders/tools/bazel-amazonlinux2023 b/builders/tools/bazel-amazonlinux2023
@@ -0,0 +1 @@
+bazel-debian