From 3b77ba18e3d6f681d6ca628b888d6050e0c66bd0 Mon Sep 17 00:00:00 2001 From: Michael Shin Date: Tue, 21 Mar 2023 16:14:54 -0400 Subject: [PATCH] Added telemetry (#47) * Added initial framework * Add handling for each message type * Fix getting current extension version * Return a new object for tracking params * Refactor to use VSCode API for telemetry * Add popup to ask for consent and update docs * Update to use better precompile directives * fix typo in README --- .gitignore | 1 + README.md | 7 ++ skyline-vscode/.vscode/tasks.json | 3 +- skyline-vscode/esbuild.mjs | 52 +++++++++++++ skyline-vscode/package.json | 37 +++++++--- .../src/analytics/AnalyticsManager.ts | 73 +++++++++++++++++++ .../src/analytics/SegmentInitializer.ts | 11 +++ skyline-vscode/src/analytics/USAGE.md | 22 ++++++ skyline-vscode/src/extension.ts | 42 ++++++++++- skyline-vscode/src/skyline_session.ts | 38 +++++++++- skyline-vscode/src/utils.ts | 10 ++- 11 files changed, 278 insertions(+), 18 deletions(-) create mode 100644 skyline-vscode/esbuild.mjs create mode 100644 skyline-vscode/src/analytics/AnalyticsManager.ts create mode 100644 skyline-vscode/src/analytics/SegmentInitializer.ts create mode 100644 skyline-vscode/src/analytics/USAGE.md diff --git a/.gitignore b/.gitignore index 3c3629e..f3db58e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +skyline-vscode/src/version.ts \ No newline at end of file diff --git a/README.md b/README.md index 3f28727..3514725 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,13 @@ After installation, please make note of the path to the `skyline` binary. To do 2. Press `Ctrl+Shift+P`, then select `Skyline` from the dropdown list. 3. Click on `Begin Analysis`. +## Disabling telemetry +If you do not want to send usage data to CentML, you can set the skyline.isTelemetryEnabled setting to "No". + +You can set the value by going to File > Preferences > Settings (On macOS: Code > Preferences > Settings), and search for telemetry. Then set the value in Skyline > Is Telemetry Enabled. This will disable all telemetry events. + +As well, DeepView respects VSCode's telemetry levels. IF telemetry.telemetryLevel is set to off, then no telemetry events will be sent to CentML, even if skyline.telemetry.enabled is set to true. If telemetry.telemetryLevel is set to error or crash, only events containing an error or errors property will be sent to CentML. + ## Development Environment Setup ### Dependencies diff --git a/skyline-vscode/.vscode/tasks.json b/skyline-vscode/.vscode/tasks.json index 3b17e53..6b9ea94 100644 --- a/skyline-vscode/.vscode/tasks.json +++ b/skyline-vscode/.vscode/tasks.json @@ -5,8 +5,7 @@ "tasks": [ { "type": "npm", - "script": "watch", - "problemMatcher": "$tsc-watch", + "script": "debug", "isBackground": true, "presentation": { "reveal": "never" diff --git a/skyline-vscode/esbuild.mjs b/skyline-vscode/esbuild.mjs new file mode 100644 index 0000000..fcf7ae7 --- /dev/null +++ b/skyline-vscode/esbuild.mjs @@ -0,0 +1,52 @@ +import * as esbuild from 'esbuild'; +import ifdefPlugin from 'esbuild-ifdef'; + +let baseOptions = { + entryPoints: ['./src/extension.js'], + bundle: true, + outfile: 'out/main.js', + external: ['vscode'], + format: 'cjs', + platform: "node" +}; + +let watchPlugin = [ + ifdefPlugin({ + variables: { + DEBUG: true + } + }) +]; + +let productionPlugin = [ + ifdefPlugin({ + variables: { + DEBUG: false + } + }) +]; + +let builds = { + 'base': { + ...baseOptions, + sourcemap: true + }, + 'debug': { + ...baseOptions, + sourcemap: true, + plugins: watchPlugin + }, + 'production': { + ...baseOptions, + minify: true, + plugins: productionPlugin + } +}; + +try { + + + await esbuild.build(builds[process.argv[2]]); +} catch (error) { + process.exit(1); +} \ No newline at end of file diff --git a/skyline-vscode/package.json b/skyline-vscode/package.json index 59e7720..66223c0 100644 --- a/skyline-vscode/package.json +++ b/skyline-vscode/package.json @@ -6,14 +6,11 @@ "description": "", "version": "0.1.0", "engines": { - "vscode": "^1.52.0" + "vscode": "^1.76.0" }, "categories": [ "Other" ], - "activationEvents": [ - "onCommand:skyline-vscode.cmd_begin_analyze" - ], "main": "./out/main.js", "contributes": { "commands": [ @@ -34,33 +31,53 @@ "type": "number", "default": 60120, "description": "Specifies the port of the profiler." + }, + "skyline.isTelemetryEnabled": { + "type": "string", + "default": "Ask me", + "enum": [ + "Ask me", + "Yes", + "No" + ], + "enumDescriptions": [ + "Prompt the user if they consent on collecting their usage data and errors.", + "Allow usage data and errors to be sent to the developer.", + "Do not allow usage data and errors to be sent to the developer." + ], + "description": "Controls DeepView telemetry and if data and errors to be sent to the developer. Note that VSCode's telemetry level is respected and takes precedence over this property." } } } }, "scripts": { - "vscode:prepublish": "npm run esbuild-base -- --minify", - "esbuild-base": "esbuild ./src/extension.ts --bundle --outfile=out/main.js --external:vscode --format=cjs --platform=node", - "esbuild": "npm run esbuild-base -- --sourcemap", - "esbuild-watch": "npm run esbuild-base -- --sourcemap --watch", - "test-compile": "tsc -p ./" + "vscode:prepublish": "npm run clean && node esbuild.mjs production", + "esbuild": "npm run clean && node esbuild.mjs base", + "watch": "npm run clean && node esbuild.mjs debug", + "clean": "rm -rf ./out", + "test": "mocha" }, "devDependencies": { "@types/glob": "^7.1.4", "@types/google-protobuf": "^3.15.5", "@types/mocha": "^9.0.0", "@types/node": "14.x", - "@types/vscode": "^1.52.0", + "@types/vscode": "^1.76.0", "@typescript-eslint/eslint-plugin": "^4.31.1", "@typescript-eslint/parser": "^4.31.1", "@vscode/test-electron": "^1.6.2", "esbuild": "^0.17.10", "eslint": "^7.32.0", "glob": "^7.1.7", + "ifdef-loader": "^2.3.2", "mocha": "^9.1.1", "typescript": "^4.4.3" }, "dependencies": { + "@segment/analytics-node": "^1.0.0-beta.23", + "@types/ws": "^8.2.0", + "bootstrap-fork": "^3.3.6", + "esbuild-ifdef": "^0.2.0", "google-protobuf": "^3.18.0", "ts-protoc-gen": "^0.15.0" } diff --git a/skyline-vscode/src/analytics/AnalyticsManager.ts b/skyline-vscode/src/analytics/AnalyticsManager.ts new file mode 100644 index 0000000..d50ccad --- /dev/null +++ b/skyline-vscode/src/analytics/AnalyticsManager.ts @@ -0,0 +1,73 @@ +import { SegmentInitializer } from "./SegmentInitializer"; +import Analytics from "@segment/analytics-node"; +import { filterObjectByKeyName } from "../utils"; +export class AnalyticsManager { + + analytics: Analytics; + hasIdentifiedUser: boolean; + userId: string; + + constructor() { + this.analytics = SegmentInitializer.initialize(); + this.hasIdentifiedUser = false; + this.userId = String(); + } + + sendEventData = (eventName: string, data?: Record) => { + this.identifyUser(data); + /// #if DEBUG + console.log("Event!"); + console.log({ + userId: this.userId, + event: eventName, + timestamp: new Date(), + properties: data + }); + /// #else + this.analytics.track({ + userId: this.userId, + event: eventName, + timestamp: new Date(), + properties: data + }); + /// #endif + }; + + sendErrorData = (error: Error, data?: Record) => { + this.identifyUser(data); + /// #if DEBUG + console.log("Error!"); + console.log({ + userId: this.userId, + event: "Client Error", + timestamp: new Date(), + properties: {... data, ...error} + }); + /// #else + this.analytics.track({ + userId: this.userId, + event: "Client Error", + timestamp: new Date(), + properties: {... data, ...error} + }); + /// #endif + }; + + closeAndFlush = () => { + this.analytics.closeAndFlush(); + }; + + identifyUser(data?: Record) { + if (!this.hasIdentifiedUser && data) { + this.userId = data["common.vscodemachineid"]; + const commonTraits = filterObjectByKeyName(data, "common."); + this.hasIdentifiedUser = true; + /// #if DEBUG + console.log("Identifying!"); + console.log({ userId: this.userId, traits:commonTraits }); + /// #else + this.analytics.identify({ userId: this.userId, traits:commonTraits }); + /// #endif + } + } +} diff --git a/skyline-vscode/src/analytics/SegmentInitializer.ts b/skyline-vscode/src/analytics/SegmentInitializer.ts new file mode 100644 index 0000000..209e4a4 --- /dev/null +++ b/skyline-vscode/src/analytics/SegmentInitializer.ts @@ -0,0 +1,11 @@ +import { Analytics, AnalyticsSettings } from '@segment/analytics-node' + + +export namespace SegmentInitializer { + export function initialize(): Analytics { + // TODO: make sure the key is inside package.json and validate it + let analyticsSettings: AnalyticsSettings = {writeKey: 'sOQXQfqVkpJxVqKbL0tbwkO6SFnpm5Ef', maxEventsInBatch: 10, flushInterval: 10000} + let analytics: Analytics = new Analytics(analyticsSettings); + return analytics; + } +} \ No newline at end of file diff --git a/skyline-vscode/src/analytics/USAGE.md b/skyline-vscode/src/analytics/USAGE.md new file mode 100644 index 0000000..abc0528 --- /dev/null +++ b/skyline-vscode/src/analytics/USAGE.md @@ -0,0 +1,22 @@ +## Usage data being collected by CentML +Only anonymous data is collected by CentML using VSCode's Telemetry API. All telemetry events are automatically sanitized to anonymize all paths (best effort) and references to the username. + +### Common data +The common data sent in all telemetry requests may contain: +- **Extension Name** `common.extname` - The extension name +- **Extension Version** `common.extversion` - The extension version +- **Machine Identifier** `common.vscodemachineid` - A common machine identifier generated by VS Code +- **Session Identifier** `common.vscodesessionid` - A session identifier generated by VS Code +- **VS Code Version** `common.vscodeversion` - The version of VS Code running the extension +- **OS** `common.os` - The OS running VS Code +- **Platform Version** `common.platformversion` - The version of the OS/Platform +- **Product** `common.product` - What Vs code is hosted in, i.e. desktop, github.dev, codespaces. +- **UI Kind** `common.uikind` - Web or Desktop indicating where VS Code is running +- **Remote Name** `common.remotename` - A name to identify the type of remote connection. `other` indicates a remote connection not from the 3 main extensions (ssh, docker, wsl). +- **Architecture** `common.nodeArch` - What architecture of node is running. i.e. arm or x86. On the web it will just say `web`. + +### Usage data +The usage data sent contains the responses given by the DeepView.Profile + +### Error data +The error data sent contains all error information thrown by the extension. diff --git a/skyline-vscode/src/extension.ts b/skyline-vscode/src/extension.ts index 3a31113..20dcffa 100644 --- a/skyline-vscode/src/extension.ts +++ b/skyline-vscode/src/extension.ts @@ -2,6 +2,11 @@ import * as vscode from 'vscode'; import {SkylineEnvironment, SkylineSession, SkylineSessionOptions} from './skyline_session'; import * as path from 'path'; +import { AnalyticsManager } from './analytics/AnalyticsManager'; + +const PRIVACY_STATEMENT_URL = "https://centml.ai/privacy-policy/"; +const OPT_OUT_INSTRUCTIONS_URL = "https://github.com/CentML/DeepView.Explore#how-to-disable-telemetry-reporting"; +const RETRY_OPTIN_DELAY_IN_MS = 60 * 60 * 1000; // 1h export function activate(context: vscode.ExtensionContext) { let sess: SkylineSession; @@ -10,6 +15,14 @@ export function activate(context: vscode.ExtensionContext) { reactProjectRoot: path.join(context.extensionPath, "react-ui") }; + let anayticsManager: AnalyticsManager = new AnalyticsManager(); + const telemetrySender: vscode.TelemetrySender = { + sendEventData: anayticsManager.sendEventData, + sendErrorData: anayticsManager.sendErrorData, + flush: anayticsManager.closeAndFlush + }; + const logger = vscode.env.createTelemetryLogger(telemetrySender); + let disposable = vscode.commands.registerCommand('skyline-vscode.cmd_begin_analyze', () => { let vsconfig = vscode.workspace.getConfiguration('skyline'); @@ -44,7 +57,9 @@ export function activate(context: vscode.ExtensionContext) { projectRoot: uri[0].fsPath, addr: vsconfig.address, port: vsconfig.port, - webviewPanel: panel + isTelemetryEnabled: isTelemetryEnabled, + webviewPanel: panel, + telemetryLogger: logger }; @@ -60,6 +75,7 @@ export function activate(context: vscode.ExtensionContext) { } startSkyline(); + showTelemetryOptInDialogIfNeeded(); }); }); @@ -69,3 +85,27 @@ export function activate(context: vscode.ExtensionContext) { export function deactivate() { } + +async function showTelemetryOptInDialogIfNeeded() { + let vsconfig = vscode.workspace.getConfiguration('skyline'); + if (vsconfig.isTelemetryEnabled === "Ask me"){ + // Pop up the message then wait + const message: string = `Help CentML improve DeepView by allowing us to collect usage data. + Read our [privacy statement](${PRIVACY_STATEMENT_URL}) + and learn how to [opt out](${OPT_OUT_INSTRUCTIONS_URL}).`; + + const retryOptin = setTimeout(showTelemetryOptInDialogIfNeeded, RETRY_OPTIN_DELAY_IN_MS); + let selection: string | undefined; + selection = await vscode.window.showInformationMessage(message, 'Yes', 'No'); + if (!selection) { + return; + } + clearTimeout(retryOptin); + vsconfig.update("isTelemetryEnabled", selection, true); + } +} + +function isTelemetryEnabled(): boolean { + let vsconfig = vscode.workspace.getConfiguration('skyline'); + return (vsconfig.isTelemetryEnabled === "Yes"); +} \ No newline at end of file diff --git a/skyline-vscode/src/skyline_session.ts b/skyline-vscode/src/skyline_session.ts index 590a687..af1d505 100644 --- a/skyline-vscode/src/skyline_session.ts +++ b/skyline-vscode/src/skyline_session.ts @@ -6,7 +6,7 @@ const fs = require('fs'); import {Socket} from 'net'; import { simpleDecoration } from './decorations'; -import { energy_component_type_mapping } from './utils'; +import { energy_component_type_mapping, getObjectKeyNameFromValue } from './utils'; const crypto = require('crypto'); const resolve = require('path').resolve; @@ -16,7 +16,9 @@ export interface SkylineSessionOptions { projectRoot: string; addr: string; port: number; - webviewPanel: vscode.WebviewPanel + isTelemetryEnabled: CallableFunction; + webviewPanel: vscode.WebviewPanel, + telemetryLogger: vscode.TelemetryLogger } export interface SkylineEnvironment { @@ -54,6 +56,10 @@ export class SkylineSession { // Environment reactProjectRoot: string; + // Analytics + isTelemetryEnabled: CallableFunction; + telemetryLogger: vscode.TelemetryLogger; + constructor(options: SkylineSessionOptions, environ: SkylineEnvironment) { console.log("SkylineSession instantiated"); @@ -76,6 +82,9 @@ export class SkylineSession { this.root_dir = options.projectRoot; this.reactProjectRoot = environ.reactProjectRoot; + this.isTelemetryEnabled = options.isTelemetryEnabled; + this.telemetryLogger = options.telemetryLogger; + this.webviewPanel.webview.onDidReceiveMessage(this.webview_handle_message.bind(this)); this.webviewPanel.onDidDispose(this.disconnect.bind(this)); this.webviewPanel.webview.html = this._getHtmlForWebview(); @@ -244,19 +253,28 @@ export class SkylineSession { this.msg_energy = msg.getEnergy(); break; }; + let eventType: string | undefined = getObjectKeyNameFromValue(pb.FromServer.PayloadCase, msg.getPayloadCase()); + eventType = eventType || "UNKNOWN"; + this.logUsage(eventType, msg.toObject()); let json_msg = await this.generateStateJson(); json_msg['message_type'] = 'analysis'; try { fs.writeFileSync('/tmp/msg.json', JSON.stringify(json_msg)); - } catch (err) { - console.error(err); + } catch (e) { + console.error(e); + if (e instanceof Error) { + this.logError(e); + } } this.webviewPanel.webview.postMessage(json_msg); } catch (e) { console.log("exception!"); console.log(message); console.log(e); + if (e instanceof Error) { + this.logError(e); + } } } @@ -463,4 +481,16 @@ export class SkylineSession { return fields; } + + logUsage(eventName: string, data?: Record){ + if (this.isTelemetryEnabled()) { + this.telemetryLogger.logUsage(eventName, data); + } + } + + logError(data?: Record){ + if (this.isTelemetryEnabled()) { + this.telemetryLogger.logError("Client Error", data); + } + } } diff --git a/skyline-vscode/src/utils.ts b/skyline-vscode/src/utils.ts index 9685250..5ad5fe9 100644 --- a/skyline-vscode/src/utils.ts +++ b/skyline-vscode/src/utils.ts @@ -13,4 +13,12 @@ export const energy_component_type_mapping = (code: number):string => { break; } return result; -} \ No newline at end of file +}; + +export const getObjectKeyNameFromValue = (obj: Object, val: any): string | undefined => { + return Object.keys(obj).find(key => obj[key as keyof Object] === val); +}; + +export const filterObjectByKeyName = (obj: Object, prefix: string): Object => { + return Object.fromEntries(Object.entries(obj).filter(([key, val])=> key.startsWith(prefix))); +}; \ No newline at end of file