fix: wallet fixes for devnet, testing and misc (#8058)

## SSH agent shenanigans

Fixes `aztec-wallet` TouchID + ECDSA via ssh in MacOS, which turns out
to be an absolute nightmare when you put docker in the mix.

Mounting the ssh-agent socket via a regular bind mount is a no-go using
the latest VirtioFS driver in Mac, but the good fellows at docker
provide a convenient "magic" path that should work:
`/run/host-services/ssh-auth.sock` (the avid reader will probably detect
the usage of the word "should" in this sentence).

Of course, the socket that gets linked is not the one defined by the
mostly universal `SSH_AUTH_SOCK` env variable like any sane person would
assume (see docker/for-mac#4242, closed due to
"inactivity", not because it's fixed!).

~~The `.ssh-hack` script takes care of that, heavily inspired by:
https://evilmartians.com/chronicles/stick-with-security-yubikey-ssh-gnupg-macos.
Then it turns out docker for mac doesn't respect permissions for the
magical variable we just symlinked, so yet another workaround is needed
(see
docker/for-mac#4242 (comment)

@charlielye suggested a way cleaner option using `socat`. It is now a
dependency unfortunately, but it avoids a lot of pain, symlinks and
general pollution of the end user's env. Basically the ssh-agent socket
is redirected with socat to the host via plain old TCP, which in turn is
used by the container (through the magical `host.docker.internal` name)
to create yet another redirection back to a container-local UNIX socket
that then gets fed to the wallet via `SSH_AUTH_SOCK`. Hooray!

## New commands

- `get-alias`: retrieves the value of any stored alias. With no args, it
prints a list of every alias stored in the wallet. It is also possible
to filter by alias type (e.g: `aztec-wallet get-alias accounts`

## Breaking changes

- Renamed `add-secret` to `create-secret` (the former made it look like
the secret had to be provided)

## Misc

- Hooks shell tests to CI, so a few flows are checked in every PR
against the sandbox.

aztec-up/bin/.aztec-run

@@ -105,7 +105,8 @@ done
 
 DOCKER_ENV="-e HOME=$HOME"
 for env in ${ENV_VARS_TO_INJECT:-}; do
-  if [ -n "${!env:-}" ]; then
+  # SSH_AUTH_SOCK must be handled separately
+  if [[ $env != "SSH_AUTH_SOCK" ]] && [[ -n "${!env:-}" ]]; then
     # First substitute any reference to localhost with our host gateway.
     env=${env//localhost/host.docker.internal}
     # Inject into container.
@@ -119,19 +120,21 @@ if [[ -z "${SKIP_PORT_ASSIGNMENT:-}" ]]; then
   port_assignment="-p $AZTEC_PORT:$AZTEC_PORT"
 fi
 
-ssh_agent_forwarding=""
-if [ -n "${SSH_AUTH_SOCK:-}" ] && [ "$(uname)" != "Darwin" ]; then
-  ssh_agent_forwarding="-v $(realpath $SSH_AUTH_SOCK):$SSH_AUTH_SOCK"
+if [[ "$ENV_VARS_TO_INJECT" == *"SSH_AUTH_SOCK"* && -n "${SSH_AUTH_SOCK:-}" ]]; then
+  warn "SSH_AUTH_SOCK is set to ${SSH_AUTH_SOCK:-}. Enabling SSH agent forwarding via socat"
+  socat TCP-LISTEN:${SSH_AUTH_SOCK_SOCAT_PORT:-12345},reuseaddr,fork UNIX-CLIENT:$SSH_AUTH_SOCK &
+  SOCAT_PID=$!
+  trap "kill -9 $SOCAT_PID" SIGINT SIGTERM EXIT
+  DOCKER_ENV+=" -e SSH_AUTH_SOCK_SOCAT_PORT=${SSH_AUTH_SOCK_SOCAT_PORT:-12345}"
 fi
 
 docker run \
   -ti \
   --rm \
   --workdir "$PWD" \
   -v $HOME:$HOME -v cache:/cache \
-  $ssh_agent_forwarding \
   $port_assignment \
   ${DOCKER_ENV:-} \
   ${DOCKER_HOST_BINDS:-} \
   ${DOCKER_USER:-} \
-  $IMAGE:$VERSION ${preserved_args[@]:-}
+  $IMAGE:$VERSION ${preserved_args[@]:-}

aztec-up/bin/aztec-install

@@ -75,6 +75,11 @@ if ! command -v docker &>/dev/null; then
   exit 1
 fi
 
+if ! command -v socat &> /dev/null; then
+  echo "Socat is not installed. Please install socat and try again."
+  exit 1
+fi
+
 # Check if Docker is running.
 if ! docker info &>/dev/null; then
   warn "Docker is not running. Please start Docker and try again."

yarn-project/Earthfile

@@ -132,9 +132,9 @@ cli-wallet-build:
 
 cli-wallet:
     FROM ubuntu:noble
-    RUN apt update && apt install nodejs curl -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    RUN apt update && apt install nodejs curl socat -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
     COPY +cli-wallet-build/usr/src /usr/src
-    ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/cli-wallet/dest/bin/index.js"]
+    ENTRYPOINT ["/usr/src/yarn-project/cli-wallet/wallet-entrypoint.sh"]
 
 export-cli-wallet:
     FROM +cli-wallet
@@ -189,8 +189,9 @@ export-aztec-faucet:
 # We care about creating a slimmed down e2e image because we have to serialize it from earthly to docker for running.
 end-to-end-prod:
     FROM +cli-base
-    RUN yarn workspaces focus @aztec/end-to-end --production && yarn cache clean
+    RUN yarn workspaces focus @aztec/end-to-end @aztec/cli-wallet --production && yarn cache clean
     COPY --dir +rollup-verifier-contract/usr/src/bb /usr/src
+    COPY --dir +build-dev/usr/src/noir-projects/noir-contracts /usr/src/noir-projects/noir-contracts
     SAVE ARTIFACT /usr/src /usr/src
 
 anvil:

yarn-project/cli-wallet/Dockerfile

@@ -1,5 +1,8 @@
 FROM aztecprotocol/yarn-project AS yarn-project
-ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/cli-wallet/dest/bin/index.js"]
+
+RUN apt update && apt install socat -y
+
+ENTRYPOINT ["/usr/src/yarn-project/cli-wallet/wallet-entrypoint.sh"]
 
 # The version has been updated in yarn-project.
 # Adding COMMIT_TAG here to rebuild versioned image.

yarn-project/cli-wallet/src/bin/index.ts

@@ -28,7 +28,23 @@ function injectInternalCommands(program: Command, log: LogFn, db: WalletDB) {
     });
 
   program
-    .command('add-secret')
+    .command('get-alias')
+    .description('Shows stored aliases')
+    .addArgument(new Argument('[alias]', 'Alias to retrieve'))
+    .action(alias => {
+      if (alias?.includes(':')) {
+        const value = db.retrieveAlias(alias);
+        log(value);
+      } else {
+        const aliases = db.listAliases(alias);
+        for (const { key, value } of aliases) {
+          log(`${key} -> ${value}`);
+        }
+      }
+    });
+
+  program
+    .command('create-secret')
     .description('Creates an aliased secret to use in other commands')
     .addOption(createAliasOption('Key to alias the secret with', false).makeOptionMandatory(true))
     .action(async (_options, command) => {

yarn-project/cli-wallet/src/cmds/add_note.ts

@@ -11,10 +11,10 @@ export async function addNote(
   storageFieldName: string,
   artifactPath: string,
   txHash: TxHash,
-  noteFields: string[],
+  noteBody: string[],
   log: LogFn,
 ) {
-  const fields = parseFields(noteFields);
+  const fields = parseFields(noteBody);
   const note = new Note(fields);
   const contractArtifact = await getContractArtifact(artifactPath, log);
 

yarn-project/cli-wallet/src/cmds/index.ts

@@ -379,15 +379,15 @@ export function injectCommands(program: Command, log: LogFn, debugLogger: DebugL
         address,
         secretKey,
         rpcUrl,
-        body: noteFields,
+        body,
         hash,
       } = options;
       const artifactPath = await artifactPathFromPromiseOrAlias(artifactPathPromise, contractAddress, db);
       const client = await createCompatibleClient(rpcUrl, debugLogger);
       const account = await createOrRetrieveAccount(client, address, db, undefined, secretKey);
       const wallet = await account.getWallet();
 
-      await addNote(wallet, address, contractAddress, noteName, storageFieldName, artifactPath, hash, noteFields, log);
+      await addNote(wallet, address, contractAddress, noteName, storageFieldName, artifactPath, hash, body, log);
     });
 
   return program;

yarn-project/cli-wallet/src/storage/wallet_db.ts

@@ -3,6 +3,7 @@ import { type LogFn } from '@aztec/foundation/log';
 import { type AztecKVStore, type AztecMap } from '@aztec/kv-store';
 
 import { type AccountType } from '../utils/accounts.js';
+import { extractECDSAPublicKeyFromBase64String } from '../utils/ecdsa.js';
 
 export const Aliases = ['accounts', 'contracts', 'artifacts', 'secrets', 'transactions'] as const;
 export type AliasType = (typeof Aliases)[number];
@@ -71,7 +72,8 @@ export class WalletDB {
     await this.#accounts.set(`${address.toString()}-sk`, secretKey.toBuffer());
     await this.#accounts.set(`${address.toString()}-salt`, salt.toBuffer());
     if (type === 'ecdsasecp256r1ssh' && publicKey) {
-      await this.storeAccountMetadata(address, 'publicSigningKey', Buffer.from(publicKey));
+      const publicSigningKey = extractECDSAPublicKeyFromBase64String(publicKey);
+      await this.storeAccountMetadata(address, 'publicSigningKey', publicSigningKey);
     }
     await this.#aliases.set('accounts:last', Buffer.from(address.toString()));
     log(`Account stored in database with alias${alias ? `es last & ${alias}` : ' last'}`);
@@ -97,13 +99,37 @@ export class WalletDB {
   }
 
   tryRetrieveAlias(arg: string) {
+    try {
+      return this.retrieveAlias(arg);
+    } catch (e) {
+      return arg;
+    }
+  }
+
+  retrieveAlias(arg: string) {
     if (Aliases.find(alias => arg.startsWith(`${alias}:`))) {
       const [type, ...alias] = arg.split(':');
       const data = this.#aliases.get(`${type}:${alias.join(':') ?? 'last'}`);
-      return data ? data.toString() : arg;
+      if (!data) {
+        throw new Error(`Could not find alias ${arg}`);
+      }
+      return data.toString();
+    } else {
+      throw new Error(`Aliases must start with one of ${Aliases.join(', ')}`);
     }
+  }
 
-    return arg;
+  listAliases(type?: AliasType) {
+    const result = [];
+    if (type && !Aliases.includes(type)) {
+      throw new Error(`Unknown alias type ${type}`);
+    }
+    for (const [key, value] of this.#aliases.entries()) {
+      if (!type || key.startsWith(`${type}:`)) {
+        result.push({ key, value: value.toString() });
+      }
+    }
+    return result;
   }
 
   async storeAccountMetadata(aliasOrAddress: AztecAddress | string, metadataKey: string, metadata: Buffer) {

yarn-project/cli-wallet/src/utils/accounts.ts

@@ -52,7 +52,7 @@ export async function createOrRetrieveAccount(
         if (!foundIdentity) {
           throw new Error(`Identity for public key ${publicKey} not found in the SSH agent`);
         }
-        publicSigningKey = extractECDSAPublicKeyFromBase64String(publicKey);
+        publicSigningKey = extractECDSAPublicKeyFromBase64String(foundIdentity.publicKey);
       } else {
         throw new Error('Public key must be provided for ECDSA SSH account');
       }

yarn-project/cli-wallet/src/utils/options/fees.ts

@@ -156,7 +156,7 @@ function parsePaymentMethod(
           } else {
             ({ claimAmount, claimSecret } = parsed);
           }
-          log(`Using Fee Juice for fee payments with claim for ${parsed.claimAmount} tokens`);
+          log(`Using Fee Juice for fee payments with claim for ${claimAmount} tokens`);
           return new FeeJuicePaymentMethodWithClaim(
             sender.getAddress(),
             BigInt(claimAmount),

yarn-project/cli-wallet/test/flows/shield_and_transfer.sh

@@ -7,7 +7,7 @@ echo
 
 aztec-wallet create-account -a main
 aztec-wallet deploy token_contract@Token --args accounts:main Test TST 18 -f main
-aztec-wallet add-secret -a shield
+aztec-wallet create-secret -a shield
 aztec-wallet send mint_private -ca contracts:last --args 42 secrets:shield:hash -f main
 aztec-wallet add-note TransparentNote pending_shields -ca contracts:last -h transactions:last -a accounts:main -b 42 secrets:shield:hash
 aztec-wallet send redeem_shield -ca contracts:last --args accounts:main 42 secrets:shield -f main

yarn-project/cli-wallet/test/test.sh

@@ -1,10 +1,15 @@
 #!/bin/bash
 set -e
 
+LOCATION=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
 NOIR_CONTRACTS_PATH=$(realpath ../../../noir-projects/noir-contracts)
 USE_DOCKER=$1
-export WALLET_DATA_DIRECTORY=$(realpath ./data)
+
+export WALLET_DATA_DIRECTORY="${LOCATION}/data"
+
 rm -rf $WALLET_DATA_DIRECTORY
+mkdir -p $WALLET_DATA_DIRECTORY
 
 COMMAND="node --no-warnings $(realpath ../dest/bin/index.js)"
 

yarn-project/cli-wallet/wallet-entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+cleanup() {
+    kill -9 $SOCAT_PID
+    rm -rf $SOCKET
+}
+
+if [[ -n "${SSH_AUTH_SOCK_SOCAT_PORT:-}" ]]; then
+    SOCKET="$HOME/.aztec/aztec-wallet-$RANDOM.sock"
+    socat UNIX-LISTEN:$SOCKET,fork TCP:host.docker.internal:${SSH_AUTH_SOCK_SOCAT_PORT} &
+    SOCAT_PID=$!
+    trap cleanup EXIT SIGKILL SIGTERM
+fi
+
+SSH_AUTH_SOCK="${SOCKET:-}" node --no-warnings /usr/src/yarn-project/cli-wallet/dest/bin/index.js $@

yarn-project/end-to-end/Earthfile

@@ -253,3 +253,6 @@ bench-prover:
 
 e2e-devnet-smoke:
   DO +E2E_COMPOSE_TEST --test=devnet/e2e_smoke.test.ts --compose_file=scripts/docker-compose-devnet.yml
+
+e2e-cli-wallet:
+  DO +E2E_COMPOSE_TEST --test=e2e_cli_wallet --compose_file=scripts/docker-compose-wallet.yml

yarn-project/end-to-end/scripts/docker-compose-wallet.yml

@@ -0,0 +1,53 @@
+version: '3'
+services:
+  fork:
+    image: aztecprotocol/foundry:de33b6af53005037b463318d2628b5cfcaf39916
+    pull_policy: always
+    entrypoint: >
+      sh -c '
+      if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then
+        exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER"
+      else
+        exec anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337
+      fi'
+    expose:
+      - '8545'
+
+  sandbox:
+    image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest}
+    command: 'start --sandbox'
+    environment:
+      DEBUG: 'aztec:*'
+      DEBUG_COLORS: 1
+      ETHEREUM_HOST: http://fork:8545
+      L1_CHAIN_ID: 31337
+      ARCHIVER_POLLING_INTERVAL_MS: 50
+      P2P_BLOCK_CHECK_INTERVAL_MS: 50
+      SEQ_TX_POLLING_INTERVAL_MS: 50
+      WS_BLOCK_CHECK_INTERVAL_MS: 50
+      PXE_BLOCK_POLLING_INTERVAL_MS: 50
+      ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
+      ENABLE_GAS: ${ENABLE_GAS:-}
+      HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-}
+    expose:
+      - '8080'
+
+  end-to-end:
+    image: aztecprotocol/end-to-end:${AZTEC_DOCKER_TAG:-latest}
+    environment:
+      DEBUG: ${DEBUG:-aztec:*}
+      DEBUG_COLORS: 1
+      ETHEREUM_HOST: http://fork:8545
+      L1_CHAIN_ID: 31337
+      PXE_URL: http://sandbox:8080
+    working_dir: /usr/src/yarn-project/cli-wallet/test
+    entrypoint: >
+      sh -c '
+        while ! nc -z sandbox 8080; do sleep 1; done;
+        ./test.sh
+      '
+    volumes:
+      - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - sandbox
+      - fork