feat: Constrain App function VKs (#9756)

Resolves #9592
 - Now contract artifacts must have VKs in their private functions
- aztec-nargo inserts the verification keys after public function
transpilation
 - We no longer derive any VK in the TX proving flow
 - App VKs are now constrained in the private kernels
 - Bootstrap generates VKs for all apps (with s3 caching)
- PXE is currently accepting any VK present in the artifact as valid: we
should explore the correct interface for this in the future and wether
PXE can use those VKs without rederiving them from ACIR

avm-transpiler/Dockerfile

@@ -9,6 +9,5 @@ RUN apt-get update && apt-get install -y git
 RUN ./scripts/bootstrap_native.sh
 
 FROM ubuntu:noble
-COPY --from=0 /usr/src/avm-transpiler/scripts/compile_then_transpile.sh /usr/src/avm-transpiler/scripts/compile_then_transpile.sh
 COPY --from=0 /usr/src/avm-transpiler/target/release/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler
 ENTRYPOINT ["/usr/src/avm-transpiler/target/release/avm-transpiler"]

avm-transpiler/Earthfile

@@ -18,7 +18,6 @@ build:
         --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \
         --build_artifacts="target"
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
-    SAVE ARTIFACT scripts/compile_then_transpile.sh
 
 format:
   FROM +source

aztec-nargo/Dockerfile

@@ -4,16 +4,21 @@ FROM aztecprotocol/noir AS built-noir
 # to get built avm-transpiler binary
 FROM aztecprotocol/avm-transpiler AS built-transpiler
 
+# to get built barretenberg binary
+FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as barretenberg
+
 
 FROM ubuntu:noble
 # Install Tini as nargo doesn't handle signals properly.
 # Install git as nargo needs it to clone.
-RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean
+RUN apt-get update && apt-get install -y git tini jq && rm -rf /var/lib/apt/lists/* && apt-get clean
 
 # Copy binaries to /usr/bin
 COPY --from=built-noir /usr/src/noir/noir-repo/target/release/nargo /usr/bin/nargo
 COPY --from=built-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler /usr/bin/avm-transpiler
+COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/bin/bb
+
 # Copy in script that calls both binaries
-COPY ./avm-transpiler/scripts/compile_then_transpile.sh /usr/src/avm-transpiler/scripts/compile_then_transpile.sh
+COPY ./aztec-nargo/compile_then_postprocess.sh /usr/src/aztec-nargo/compile_then_postprocess.sh
 
-ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/avm-transpiler/scripts/compile_then_transpile.sh"]
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/aztec-nargo/compile_then_postprocess.sh"]

aztec-nargo/Earthfile

@@ -4,17 +4,20 @@ run:
     FROM ubuntu:noble
     # Install Tini as nargo doesn't handle signals properly.
     # Install git as nargo needs it to clone.
-    RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean
+    RUN apt-get update && apt-get install -y git tini jq && rm -rf /var/lib/apt/lists/* && apt-get clean
 
     # Copy binaries to /usr/bin
     COPY ../noir+nargo/nargo /usr/bin/nargo
     COPY ../avm-transpiler+build/avm-transpiler /usr/bin/avm-transpiler
+    COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/bin/bb
+
     # Copy in script that calls both binaries
-    COPY ../avm-transpiler+build/compile_then_transpile.sh /usr/bin/compile_then_transpile.sh
+    COPY ./compile_then_postprocess.sh /usr/bin/compile_then_postprocess.sh
 
     ENV PATH "/usr/bin:${PATH}"
-    ENTRYPOINT ["/usr/bin/tini", "--", "/usr/bin/compile_then_transpile.sh"]
+    ENTRYPOINT ["/usr/bin/tini", "--", "/usr/bin/compile_then_postprocess.sh"]
     SAVE IMAGE aztecprotocol/aztec-nargo
+    SAVE ARTIFACT /usr/bin/compile_then_postprocess.sh /aztec-nargo
 
 export-aztec-nargo:
   FROM +run

aztec-nargo/compile_then_postprocess.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# This is a wrapper script for nargo.
+# Pass any args that you'd normally pass to nargo.
+# If the first arg is "compile",
+# run nargo and then postprocess any created artifacts.
+#
+# Usage: compile_then_postprocess.sh [nargo args]
+set -eu
+
+NARGO=${NARGO:-nargo}
+TRANSPILER=${TRANSPILER:-avm-transpiler}
+BB=${BB:-bb}
+
+if [ "${1:-}" != "compile" ]; then
+  # if not compiling, just pass through to nargo verbatim
+  $NARGO $@
+  exit $?
+fi
+shift # remove the compile arg so we can inject --show-artifact-paths
+
+# Forward all arguments to nargo, tee output to console
+artifacts_to_process=$($NARGO compile --show-artifact-paths $@ | tee /dev/tty | grep -oP 'Saved contract artifact to: \K.*')
+
+# NOTE: the output that is teed to /dev/tty will normally not be redirectable by the caller.
+# If the script is run via docker, however, the user will see this output on stdout and will be able to redirect.
+
+# Postprocess each artifact
+# `$artifacts_to_process` needs to be unquoted here, otherwise it will break if there are multiple artifacts
+for artifact in $artifacts_to_process; do
+  # transpiler input and output files are the same (modify in-place)
+  $TRANSPILER "$artifact" "$artifact"
+  artifact_name=$(basename "$artifact")
+  echo "Generating verification keys for functions in $artifact_name"
+  # See contract_artifact.ts (getFunctionType) for reference
+  private_fn_indices=$(jq -r '.functions | to_entries | map(select((.value.custom_attributes | contains(["public"]) | not) and (.value.is_unconstrained == false))) | map(.key) | join(" ")' $artifact)
+  for fn_index in $private_fn_indices; do
+    fn_name=$(jq -r ".functions[$fn_index].name" $artifact)
+    fn_artifact=$(jq -r ".functions[$fn_index]" $artifact)
+    fn_artifact_path="$artifact.function_artifact_$fn_index.json"
+    echo $fn_artifact > $fn_artifact_path
+
+    echo "Generating verification key for function $fn_name"
+    # BB outputs the verification key to stdout as raw bytes, however, we need to base64 encode it before storing it in the artifact
+    verification_key=$($BB write_vk_mega_honk -h -b ${fn_artifact_path} -o - | base64)
+    rm $fn_artifact_path
+    jq ".functions[$fn_index].verification_key = \"$verification_key\"" $artifact > $artifact.tmp
+    mv $artifact.tmp $artifact
+  done
+done

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -275,6 +275,15 @@ WASM_EXPORT void acir_vk_as_fields_ultra_honk(uint8_t const* vk_buf, fr::vec_out
 {
     using VerificationKey = UltraFlavor::VerificationKey;
 
+    auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(vk_buf));
+    std::vector<bb::fr> vkey_as_fields = verification_key->to_field_elements();
+    *out_vkey = to_heap_buffer(vkey_as_fields);
+}
+
+WASM_EXPORT void acir_vk_as_fields_mega_honk(uint8_t const* vk_buf, fr::vec_out_buf out_vkey)
+{
+    using VerificationKey = MegaFlavor::VerificationKey;
+
     auto verification_key = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(vk_buf));
     std::vector<bb::fr> vkey_as_fields = verification_key->to_field_elements();
     *out_vkey = to_heap_buffer(vkey_as_fields);

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.hpp

@@ -93,4 +93,6 @@ WASM_EXPORT void acir_write_vk_ultra_honk(uint8_t const* acir_vec, bool const* r
 
 WASM_EXPORT void acir_proof_as_fields_ultra_honk(uint8_t const* proof_buf, fr::vec_out_buf out);
 
-WASM_EXPORT void acir_vk_as_fields_ultra_honk(uint8_t const* vk_buf, fr::vec_out_buf out_vkey);
+WASM_EXPORT void acir_vk_as_fields_ultra_honk(uint8_t const* vk_buf, fr::vec_out_buf out_vkey);
+
+WASM_EXPORT void acir_vk_as_fields_mega_honk(uint8_t const* vk_buf, fr::vec_out_buf out_vkey);

barretenberg/exports.json

@@ -917,5 +917,21 @@
       }
     ],
     "isAsync": false
+  },
+  {
+    "functionName": "acir_vk_as_fields_mega_honk",
+    "inArgs": [
+      {
+        "name": "vk_buf",
+        "type": "const uint8_t *"
+      }
+    ],
+    "outArgs": [
+      {
+        "name": "out_vkey",
+        "type": "fr::vec_out_buf"
+      }
+    ],
+    "isAsync": false
   }
 ]

barretenberg/ts/src/barretenberg_api/index.ts

@@ -604,6 +604,18 @@ export class BarretenbergApi {
     const out = result.map((r, i) => outTypes[i].fromBuffer(r));
     return out[0];
   }
+
+  async acirVkAsFieldsMegaHonk(vkBuf: Uint8Array): Promise<Fr[]> {
+    const inArgs = [vkBuf].map(serializeBufferable);
+    const outTypes: OutputType[] = [VectorDeserializer(Fr)];
+    const result = await this.wasm.callWasmExport(
+      'acir_vk_as_fields_mega_honk',
+      inArgs,
+      outTypes.map(t => t.SIZE_IN_BYTES),
+    );
+    const out = result.map((r, i) => outTypes[i].fromBuffer(r));
+    return out[0];
+  }
 }
 export class BarretenbergApiSync {
   constructor(protected wasm: BarretenbergWasm) {}
@@ -1181,4 +1193,16 @@ export class BarretenbergApiSync {
     const out = result.map((r, i) => outTypes[i].fromBuffer(r));
     return out[0];
   }
+
+  acirVkAsFieldsMegaHonk(vkBuf: Uint8Array): Fr[] {
+    const inArgs = [vkBuf].map(serializeBufferable);
+    const outTypes: OutputType[] = [VectorDeserializer(Fr)];
+    const result = this.wasm.callWasmExport(
+      'acir_vk_as_fields_mega_honk',
+      inArgs,
+      outTypes.map(t => t.SIZE_IN_BYTES),
+    );
+    const out = result.map((r, i) => outTypes[i].fromBuffer(r));
+    return out[0];
+  }
 }

boxes/Dockerfile

@@ -2,17 +2,23 @@
 FROM aztecprotocol/aztec AS aztec
 FROM aztecprotocol/noir as noir
 FROM aztecprotocol/noir-projects as noir-projects
+FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as barretenberg
+FROM aztecprotocol/avm-transpiler AS transpiler
 
 # We need yarn. Start fresh container.
 FROM node:18.19.0
-RUN apt update && apt install netcat-openbsd
+RUN apt update && apt install netcat-openbsd jq
 COPY --from=aztec /usr/src /usr/src
-COPY --from=noir /usr/src/noir/noir-repo/target/release/nargo /usr/src/noir/noir-repo/target/release/nargo
 COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr
 COPY --from=noir-projects /usr/src/noir-projects/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types
+# Copy binaries to /usr/bin
+COPY --from=noir /usr/src/noir/noir-repo/target/release/nargo /usr/src/noir/noir-repo/target/release/nargo
+COPY --from=transpiler /usr/src/avm-transpiler/target/release/avm-transpiler /usr/bin/avm-transpiler
+COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/bin/bb
+
 WORKDIR /usr/src/boxes
 COPY . .
-ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo
+ENV AZTEC_NARGO=/usr/aztec-nargo/compile_then_postprocess.sh
 ENV AZTEC_BUILDER=/usr/src/yarn-project/builder/aztec-builder-dest
 RUN yarn
 RUN npx -y playwright@1.42 install --with-deps

boxes/Earthfile

@@ -31,10 +31,15 @@ build:
     COPY ../noir/+nargo/nargo /usr/src/noir/noir-repo/target/release/nargo
     COPY ../noir-projects/+build/aztec-nr /usr/src/noir-projects/aztec-nr
     COPY ../noir-projects/+build/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types
-
+    COPY ../avm-transpiler+build/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler
+    COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+    COPY ../aztec-nargo+run/aztec-nargo /usr/src/aztec-nargo
     WORKDIR /usr/src/boxes
 
-    ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo
+    ENV NARGO=/usr/src/noir/noir-repo/target/release/nargo
+    ENV TRANSPILER=/usr/src/avm-transpiler/target/release/avm-transpiler
+    ENV BB=/usr/src/barretenberg/cpp/build/bin/bb
+    ENV AZTEC_NARGO=/usr/src/aztec-nargo
     ENV AZTEC_BUILDER=/usr/src/yarn-project/builder/aztec-builder-dest
     COPY . .
     RUN yarn build

build_manifest.yml

@@ -55,11 +55,8 @@ avm-transpiler:
 aztec-nargo:
   buildDir: .
   dockerfile: aztec-nargo/Dockerfile
-  rebuildPatterns:
-    - ^aztec-nargo/
-    - ^avm-transpiler/
-    - ^noir/
   dependencies:
+    - barretenberg-x86_64-linux-clang
     - avm-transpiler
     - noir
   multiarch: buildx
@@ -250,8 +247,10 @@ boxes:
   buildDir: boxes
   dependencies:
     - aztec
-    - noir
     - noir-projects
+    - noir
+    - avm-transpiler
+    - barretenberg-x86_64-linux-clang
   runDependencies:
     - aztec
 

noir-projects/noir-contracts/bootstrap.sh

@@ -19,20 +19,15 @@ echo "Compiling contracts..."
 NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo}
 $NARGO compile --silence-warnings --inliner-aggressiveness 0
 
-echo "Generating protocol contract vks..."
+echo "Transpiling contracts..."
+scripts/transpile.sh
+
+echo "Postprocessing contracts..."
 BB_HASH=${BB_HASH:-$(cd ../../ && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)}
 echo Using BB hash $BB_HASH
-vkDir="./target/keys"
-mkdir -p $vkDir
+tempDir="./target/tmp"
+mkdir -p $tempDir
 
-protocol_contracts=$(jq -r '.[]' "./protocol_contracts.json")
-for contract in $protocol_contracts; do
-    artifactPath="./target/$contract.json"
-    fnNames=$(jq -r '.functions[] | select(.custom_attributes | index("private")) | .name' "$artifactPath")
-    for fnName in $fnNames; do
-      BB_HASH=$BB_HASH node ../scripts/generate_vk_json.js "$artifactPath" "$vkDir" "$fnName"
-    done
+for artifactPath in "./target"/*.json; do
+    BB_HASH=$BB_HASH node ./scripts/postprocess_contract.js "$artifactPath" "$tempDir"
 done
-
-echo "Transpiling contracts..."
-scripts/transpile.sh