From 2ee1a16d3c59e6f447c8c288ab37b97495df368d Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 17 Oct 2024 17:09:17 -0700 Subject: [PATCH] Report cell execution status to Foyle. * This updates the extension to report the status of the cell execution to Foyle when reporting execution events. * This is highly valuable signal for the AI. This will help us from learning from broken/invalid commands. * Fix jlewi/foyle#310 --- package-lock.json | 14 +++++++------- package.json | 4 ++-- src/extension/ai/events.ts | 13 ++++++++++--- src/extension/kernel.ts | 8 ++++---- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 098240e9e..31e43ca7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,8 +15,8 @@ "@aws-sdk/client-eks": "^3.667.0", "@aws-sdk/credential-providers": "^3.667.0", "@buf/grpc_grpc.community_timostamm-protobuf-ts": "^2.9.4-20240904201038-e0425cfebb28.4", - "@buf/jlewi_foyle.bufbuild_es": "^1.10.0-20241002214001-8c67df189216.1", - "@buf/jlewi_foyle.connectrpc_es": "^1.5.0-20241002214001-8c67df189216.1", + "@buf/jlewi_foyle.bufbuild_es": "^1.10.0-20241018000047-325e896f72bd.1", + "@buf/jlewi_foyle.connectrpc_es": "^1.5.0-20241018000047-325e896f72bd.1", "@buf/stateful_runme.community_timostamm-protobuf-ts": "^2.9.4-20240913234806-45813f39881a.4", "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-node": "^1.5.0", @@ -2402,8 +2402,8 @@ } }, "node_modules/@buf/jlewi_foyle.bufbuild_es": { - "version": "1.10.0-20241002214001-8c67df189216.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.bufbuild_es/-/jlewi_foyle.bufbuild_es-1.10.0-20241002214001-8c67df189216.1.tgz", + "version": "1.10.0-20241018000047-325e896f72bd.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.bufbuild_es/-/jlewi_foyle.bufbuild_es-1.10.0-20241018000047-325e896f72bd.1.tgz", "dependencies": { "@buf/bufbuild_protovalidate.bufbuild_es": "1.10.0-20240212200630-3014d81c3a48.1", "@buf/stateful_runme.bufbuild_es": "1.10.0-20240913234806-45813f39881a.1" @@ -2413,11 +2413,11 @@ } }, "node_modules/@buf/jlewi_foyle.connectrpc_es": { - "version": "1.5.0-20241002214001-8c67df189216.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.connectrpc_es/-/jlewi_foyle.connectrpc_es-1.5.0-20241002214001-8c67df189216.1.tgz", + "version": "1.5.0-20241018000047-325e896f72bd.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/jlewi_foyle.connectrpc_es/-/jlewi_foyle.connectrpc_es-1.5.0-20241018000047-325e896f72bd.1.tgz", "dependencies": { "@buf/bufbuild_protovalidate.connectrpc_es": "1.5.0-20240212200630-3014d81c3a48.1", - "@buf/jlewi_foyle.bufbuild_es": "1.10.0-20241002214001-8c67df189216.1", + "@buf/jlewi_foyle.bufbuild_es": "1.10.0-20241018000047-325e896f72bd.1", "@buf/stateful_runme.connectrpc_es": "1.5.0-20240913234806-45813f39881a.1" }, "peerDependencies": { diff --git a/package.json b/package.json index d6397f29c..f8ea5af38 100644 --- a/package.json +++ b/package.json @@ -1272,8 +1272,8 @@ "@aws-sdk/client-eks": "^3.667.0", "@aws-sdk/credential-providers": "^3.667.0", "@buf/grpc_grpc.community_timostamm-protobuf-ts": "^2.9.4-20240904201038-e0425cfebb28.4", - "@buf/jlewi_foyle.bufbuild_es": "^1.10.0-20241002214001-8c67df189216.1", - "@buf/jlewi_foyle.connectrpc_es": "^1.5.0-20241002214001-8c67df189216.1", + "@buf/jlewi_foyle.bufbuild_es": "^1.10.0-20241018000047-325e896f72bd.1", + "@buf/jlewi_foyle.connectrpc_es": "^1.5.0-20241018000047-325e896f72bd.1", "@buf/stateful_runme.community_timostamm-protobuf-ts": "^2.9.4-20240913234806-45813f39881a.4", "@connectrpc/connect": "^1.4.0", "@connectrpc/connect-node": "^1.5.0", diff --git a/src/extension/ai/events.ts b/src/extension/ai/events.ts index a4924237c..71b9e4e5c 100644 --- a/src/extension/ai/events.ts +++ b/src/extension/ai/events.ts @@ -4,6 +4,7 @@ import { LogEventsRequest, LogEventType, LogEvent, + LogEvent_ExecuteStatus, } from '@buf/jlewi_foyle.bufbuild_es/foyle/v1alpha1/agent_pb' import * as vscode from 'vscode' import { ulid } from 'ulidx' @@ -17,7 +18,7 @@ import * as converters from './converters' // Interface for the event reporter // This allows us to swap in a null op logger when AI isn't enabled export interface IEventReporter { - reportExecution(cell: vscode.NotebookCell): Promise + reportExecution(cell: vscode.NotebookCell, status: boolean): Promise reportEvents(events: LogEvent[]): Promise } @@ -31,7 +32,7 @@ export class EventReporter implements IEventReporter { this.log = getLogger('AIEventReporter') } - async reportExecution(cell: vscode.NotebookCell) { + async reportExecution(cell: vscode.NotebookCell, executionSuccess: boolean) { const contextCells: vscode.NotebookCell[] = [] // Include some previous cells as context. @@ -58,6 +59,12 @@ export class EventReporter implements IEventReporter { event.type = LogEventType.EXECUTE event.cells = cells event.contextId = SessionManager.getManager().getID() + + if (executionSuccess) { + event.executeStatus = LogEvent_ExecuteStatus.SUCCEEDED + } else { + event.executeStatus = LogEvent_ExecuteStatus.FAILED + } return this.reportEvents([event]) } @@ -77,7 +84,7 @@ export class EventReporter implements IEventReporter { // NullOpEventReporter is a null op implementation of the event reporter export class NullOpEventReporter implements IEventReporter { - async reportExecution(_cell: vscode.NotebookCell) { + async reportExecution(_cell: vscode.NotebookCell, _status: boolean) { // Do nothing } diff --git a/src/extension/kernel.ts b/src/extension/kernel.ts index 0f07b1053..c5320f42b 100644 --- a/src/extension/kernel.ts +++ b/src/extension/kernel.ts @@ -773,8 +773,6 @@ export class Kernel implements Disposable { } TelemetryReporter.sendTelemetryEvent('cell.startExecute') - // todo(sebastian): rewrite to use non-blocking impl - const execCellReport = getEventReporter().reportExecution(cell) runmeExec.start(Date.now()) const annotations = getAnnotations(cell) @@ -821,10 +819,12 @@ export class Kernel implements Disposable { successfulCellExecution = false log.error('Error executing cell', e.message) window.showErrorMessage(e.message) - } finally { - await execCellReport } + // todo(sebastian): rewrite to use non-blocking impl + const execCellReport = getEventReporter().reportExecution(cell, successfulCellExecution) + await execCellReport + TelemetryReporter.sendTelemetryEvent('cell.endExecute', { 'cell.success': successfulCellExecution?.toString(), 'cell.mimeType': annotations.mimeType,