chore: Export trace data to google cloud

spartan/aztec-network/templates/boot-node.yaml

@@ -180,6 +180,8 @@ spec:
               value: "{{ .Values.bootNode.logLevel }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: P2P_ENABLED
               value: "{{ .Values.bootNode.p2p.enabled }}"
             - name: COINBASE

spartan/aztec-network/templates/faucet.yaml

@@ -79,6 +79,8 @@ spec:
               value: "{{ .Values.faucet.l1Assets }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: LOG_LEVEL
               value: "{{ .Values.faucet.logLevel }}"
           ports:

spartan/aztec-network/templates/prover-agent.yaml

@@ -94,6 +94,8 @@ spec:
               value: "{{ .Values.proverAgent.logLevel }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: PROVER_REAL_PROOFS
               value: "{{ .Values.aztec.realProofs }}"
             - name: PROVER_AGENT_COUNT

spartan/aztec-network/templates/prover-broker.yaml

@@ -76,6 +76,8 @@ spec:
               value: "{{ .Values.proverBroker.logLevel }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: PROVER_BROKER_POLL_INTERVAL_MS
               value: "{{ .Values.proverBroker.pollIntervalMs }}"
             - name: PROVER_BROKER_JOB_TIMEOUT_MS

spartan/aztec-network/templates/prover-node.yaml

@@ -125,6 +125,8 @@ spec:
               value: "{{ .Values.proverNode.logLevel }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: PROVER_REAL_PROOFS
               value: "{{ .Values.aztec.realProofs }}"
             - name: PROVER_AGENT_COUNT

spartan/aztec-network/templates/pxe.yaml

@@ -93,6 +93,8 @@ spec:
               value: "{{ .Values.pxe.service.nodePort }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: LOG_LEVEL
               value: "{{ .Values.pxe.logLevel }}"
             - name: PXE_PROVER_ENABLED

spartan/aztec-network/templates/transaction-bot.yaml

@@ -87,6 +87,8 @@ spec:
               value: "{{ .Values.bot.service.nodePort }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: LOG_LEVEL
               value: "{{ .Values.bot.logLevel }}"
             - name: BOT_PRIVATE_KEY

spartan/aztec-network/templates/validator.yaml

@@ -162,6 +162,8 @@ spec:
               value: "{{ .Values.validator.logLevel }}"
             - name: LOG_JSON
               value: "1"
+            - name: LOG_GCLOUD
+              value: "{{ .Values.telemetry.gcloud }}"
             - name: P2P_ENABLED
               value: "{{ .Values.validator.p2p.enabled }}"
             - name: VALIDATOR_DISABLED

spartan/aztec-network/values.yaml

@@ -13,6 +13,7 @@ network:
 
 telemetry:
   enabled: false
+  gcloud:
   otelCollectorEndpoint:
 
 images:

spartan/terraform/deploy-release/main.tf

@@ -100,6 +100,11 @@ resource "helm_release" "aztec-gke-cluster" {
     value = var.L1_DEPLOYMENT_SALT
   }
 
+  set {
+    name  = "telemetry.gcloud"
+    value = "true"
+  }
+
   # Setting timeout and wait conditions
   timeout       = 1200 # 20 minutes in seconds
   wait          = true

yarn-project/foundation/src/config/env_var.ts

@@ -57,6 +57,7 @@ export type EnvVar =
   | 'L2_QUEUE_SIZE'
   | 'LOG_ELAPSED_TIME'
   | 'LOG_JSON'
+  | 'LOG_GCLOUD'
   | 'LOG_MULTILINE'
   | 'LOG_LEVEL'
   | 'MNEMONIC'

yarn-project/foundation/src/log/gcloud-logger.ts

@@ -0,0 +1,71 @@
+import { type pino } from 'pino';
+
+/* eslint-disable camelcase */
+
+const GOOGLE_CLOUD_TRACE_ID = 'logging.googleapis.com/trace';
+const GOOGLE_CLOUD_SPAN_ID = 'logging.googleapis.com/spanId';
+const GOOGLE_CLOUD_TRACE_SAMPLED = 'logging.googleapis.com/trace_sampled';
+
+/**
+ * Pino configuration for google cloud observability. Tweaks message and timestamp,
+ * adds trace context attributes, and injects severity level.
+ * Adapted from https://cloud.google.com/trace/docs/setup/nodejs-ot#config-structured-logging.
+ */
+export const GoogleCloudLoggerConfig = {
+  messageKey: 'message',
+  // Same as pino.stdTimeFunctions.isoTime but uses "timestamp" key instead of "time"
+  timestamp(): string {
+    return `,"timestamp":"${new Date(Date.now()).toISOString()}"`;
+  },
+  formatters: {
+    log(object: Record<string, unknown>): Record<string, unknown> {
+      // Add trace context attributes following Cloud Logging structured log format described
+      // in https://cloud.google.com/logging/docs/structured-logging#special-payload-fields
+      const { trace_id, span_id, trace_flags, ...rest } = object;
+
+      if (trace_id && span_id) {
+        return {
+          [GOOGLE_CLOUD_TRACE_ID]: trace_id,
+          [GOOGLE_CLOUD_SPAN_ID]: span_id,
+          [GOOGLE_CLOUD_TRACE_SAMPLED]: trace_flags ? trace_flags === '01' : undefined,
+          trace_flags, // Keep the original trace_flags for otel-pino-stream
+          ...rest,
+        };
+      }
+      return object;
+    },
+    level(label: string, level: number): object {
+      // Inspired by https://github.com/pinojs/pino/issues/726#issuecomment-605814879
+      // Severity labels https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
+      let severity: string;
+
+      switch (label as pino.Level | keyof typeof customLevels) {
+        case 'trace':
+        case 'debug':
+          severity = 'DEBUG';
+          break;
+        case 'verbose':
+        case 'info':
+          severity = 'INFO';
+          break;
+        case 'warn':
+          severity = 'WARNING';
+          break;
+        case 'error':
+          severity = 'ERROR';
+          break;
+        case 'fatal':
+          severity = 'CRITICAL';
+          break;
+        default:
+          severity = 'DEFAULT';
+          break;
+      }
+
+      return { severity, level };
+    },
+  },
+} satisfies pino.LoggerOptions;
+
+// Define custom logging levels for pino. Duplicate from pino-logger.ts.
+const customLevels = { verbose: 25 };

yarn-project/foundation/src/log/pino-logger.ts

@@ -5,6 +5,7 @@ import { type Writable } from 'stream';
 import { inspect } from 'util';
 
 import { compactArray } from '../collection/array.js';
+import { GoogleCloudLoggerConfig } from './gcloud-logger.js';
 import { getLogLevelFromFilters, parseEnv } from './log-filters.js';
 import { type LogLevel } from './log-levels.js';
 import { type LogData, type LogFn } from './log_fn.js';
@@ -70,44 +71,20 @@ const [logLevel, logFilters] = parseEnv(process.env.LOG_LEVEL, defaultLogLevel);
 // Define custom logging levels for pino.
 const customLevels = { verbose: 25 };
 
-// inspired by https://github.com/pinojs/pino/issues/726#issuecomment-605814879
-const levelToSeverityFormatter = (label: string, level: number): object => {
-  // Severity labels https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
-  let severity: string;
-
-  switch (label as pino.Level | keyof typeof customLevels) {
-    case 'trace':
-    case 'debug':
-      severity = 'DEBUG';
-      break;
-    case 'verbose':
-    case 'info':
-      severity = 'INFO';
-      break;
-    case 'warn':
-      severity = 'WARNING';
-      break;
-    case 'error':
-      severity = 'ERROR';
-      break;
-    case 'fatal':
-      severity = 'CRITICAL';
-      break;
-    default:
-      severity = 'DEFAULT';
-      break;
-  }
+// Do we want to log in JSON format?
+const logJson = ['1', 'true', 'TRUE'].includes(process.env.LOG_JSON ?? '');
 
-  return { severity, level };
-};
+// Are we in google cloud? If so, format logs for its logs and traces explorer.
+const logGoogleCloud = ['1', 'true', 'TRUE'].includes(process.env.LOG_GCLOUD ?? '');
 
+// Define global options for pino.
 const pinoOpts: pino.LoggerOptions<keyof typeof customLevels> = {
   customLevels,
   useOnlyCustomLevels: false,
   level: logLevel,
-  formatters: {
-    level: levelToSeverityFormatter,
-  },
+  // While it'd be great to set the google cloud formatters for the stdio transport only,
+  // pino requires that we set formatters at the logger level instead.
+  ...(logGoogleCloud ? GoogleCloudLoggerConfig : {}),
 };
 
 export const levels = {
@@ -170,7 +147,7 @@ function makeLogger() {
     // Regular nodejs with transports on worker thread, using pino-pretty for console logging if LOG_JSON
     // is not set, and an optional OTLP transport if the OTLP endpoint is provided.
     const targets: pino.TransportSingleOptions[] = compactArray([
-      ['1', 'true', 'TRUE'].includes(process.env.LOG_JSON ?? '') ? stdioTransport : prettyTransport,
+      logJson ? stdioTransport : prettyTransport,
       otlpEndpoint ? otelTransport : undefined,
     ]);
     return pino(pinoOpts, pino.transport({ targets, levels: levels.values }));

yarn-project/telemetry-client/src/otel.ts

@@ -126,7 +126,6 @@ export class OpenTelemetryClient implements TelemetryClient {
   public static async createAndStart(config: TelemetryClientConfig, log: Logger): Promise<OpenTelemetryClient> {
     const resource = await getOtelResource();
 
-    // TODO(palla/log): Should we show traces as logs in stdout when otel collection is disabled?
     const tracerProvider = new NodeTracerProvider({
       resource,
       spanProcessors: config.tracesCollectorUrl

yarn-project/telemetry-client/src/vendor/otel-pino-stream.ts

@@ -222,6 +222,10 @@ export class OTelPinoStream extends Writable {
         // [aztec] They are not redundant, we depend on them for correlation.
         // The instrumentation package seems to be adding these fields via a custom hook.
         // We push them from the logger module in foundation, so we don't want to clear them here.
+        // We do rename the google-cloud specific fields though, back to their expected names.
+        ['logging.googleapis.com/trace']: trace_id,
+        ['logging.googleapis.com/spanId']: span_id,
+        ['logging.googleapis.com/trace_sampled']: _trace_flags,
 
         ...attributes
       } = recObj;
@@ -232,6 +236,11 @@ export class OTelPinoStream extends Writable {
         timestamp = Date.now();
       }
 
+      if (span_id && trace_id) {
+        attributes['trace_id'] = trace_id;
+        attributes['span_id'] = span_id;
+      }
+
       // This avoids a possible subtle bug when a Pino logger uses
       // `time: pino.stdTimeFunctions.unixTime` and logs in the first half-second
       // since process start. The rounding involved results in: