Created plugin-ext-metrics for creating metrics from language plugins

.travis.yml

@@ -46,6 +46,7 @@ cache:
   - packages/outline-view/node_modules
   - packages/output/node_modules
   - packages/plugin-dev/node_modules
+  - packages/plugin-ext-metrics/node_modules
   - packages/plugin-ext-vscode/node_modules
   - packages/plugin-ext/node_modules
   - packages/plugin/node_modules

examples/browser/package.json

@@ -43,6 +43,7 @@
     "@theia/output": "^0.11.0",
     "@theia/plugin-dev": "^0.11.0",
     "@theia/plugin-ext": "^0.11.0",
+    "@theia/plugin-ext-metrics": "^0.11.0",
     "@theia/plugin-ext-vscode": "^0.11.0",
     "@theia/preferences": "^0.11.0",
     "@theia/preview": "^0.11.0",

packages/plugin-ext-metrics/.gitignore

@@ -0,0 +1,25 @@
+.DS_Store
+node_modules
+build
+lib
+*.log
+.idea
+.metadata
+*.iml
+jdt.ls-java-project
+lerna-debug.log
+.nyc_output
+coverage
+errorShots
+examples/*/src-gen
+examples/*/webpack.config.js
+.browser_modules
+**/docs/api
+package-backup.json
+.history
+.Trash-*
+packages/plugin/typedoc
+plugins
+gh-pages
+.vscode/ipch
+dev-packages/electron/compile_commands.json

packages/plugin-ext-metrics/README.md

@@ -0,0 +1,38 @@
+
+# Theia Plugin Extension Metrics
+This extension provides metrics for the theia plugin extension in the prometheus format.
+
+### What Metrics it Detects
+1. Detects errors in languages that are registered directly with monaco (E.g. if an error happens here: https://github.com/microsoft/vscode-extension-samples/blob/master/completions-sample/src/extension.ts#L11 it will be reported).
+
+2. Detects errors that are logged directly to the output channel for a specific vscode extension that uses a language server. These errors can only be reported via their id that is registered with the vscode-languageclient library. E.g. "YAML Support", "XML Support", etc
+
+### Limitations & Drawbacks
+Due to the limitations of the vscode-languageclient library (see https://github.com/microsoft/vscode-languageserver-node/issues/517) we are unable to process errors that come from the language server directly, instead we need to use the output channel. The output channel is great because it allows us to work around limitations of the vscode-languageclient library and still get metrics but it still has some drawbacks:
+
+1. Everytime a language server request is resolve it counts as a success. This is because the vscode-languageclient always sends back a resolved promise even when the promise is actually rejected. The only time you can get an error is by extracing data from the output channel using a regex and connecting it back to the successes that were counted earlier. This has a few consequences:
+    1. If the errors logged are not matched by the regex we have no way to know where the error occured and thus we can't link the error back to a language server method. That means that the metric we created will always show that its working 100% correctly, even though its not. This is why we currently cannot get metrics for the vscode-java extension (it uses a non-standard output format).
+
+2. You need to manually add a mapping of the output channel id to the vscode extension id, otherwise when the request is logged to the output channel it doesn't know which vscode extension it should associate itself with. There is no way around this because the output channel id is registered in the vscode-languageclient library inside of the vscode extension and not in something like the vscode-extensions package.json.
+
+### Implementation
+The browser side of this extension rebinds key parts of the plugin-ext allowing us to abstract relevant metrics at certain points.
+
+The browser then collects all these key metrics in the plugin-metrics-creator class.
+
+Once we have all the data we want, we need to transfer the data from the frontend to the backend so that our new metrics are display on /metrics endpoint. This communication is done via JSON-RPC where the PluginMetrics interface acts as as common way to pass information between the frontend and the backend. To learn more see [1]
+
+The plugin-metrics-extractor will set the plugin metrics every 5 seconds [2] via pluginMetrics.setMetrics(metrics: string).
+
+Then, every 5 seconds [2] the backend will check the plugin metrics via pluginMetrics.getMetrics() to see what the contents of the metrics are at that time.
+
+Then, when you load up the /metrics endpoint you will see the new language metrics.
+
+[1] - [https://www.theia-ide.org/docs/json_rpc](https://www.theia-ide.org/docs/json_rpc)
+
+[2] - This is configurable and lives in common/metrics-protocol.ts 
+
+## License
+
+-  [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/)
+-  [一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp)

packages/plugin-ext-metrics/compile.tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../configs/base.tsconfig",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "lib"
+  },
+  "include": [
+    "src"
+  ]
+}

packages/plugin-ext-metrics/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@theia/plugin-ext-metrics",
+  "version": "0.11.0",
+  "description": "Theia - Plugin Extension Metrics",
+  "dependencies": {
+    "@theia/core": "^0.11.0",
+    "@theia/languages": "^0.11.0",
+    "@theia/metrics": "^0.11.0",
+    "@theia/plugin-ext": "^0.11.0",
+    "vscode-languageserver-protocol": "^3.14.1",
+    "@theia/plugin": "^0.11.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "theiaExtensions": [
+    {
+      "frontend": "lib/browser/plugin-metrics-frontend-module",
+      "backend": "lib/node/plugin-ext-metrics-backend-module"
+    }
+  ],
+  "keywords": [
+    "theia-extension"
+  ],
+  "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/theia-ide/theia.git"
+  },
+  "bugs": {
+    "url": "https://github.com/theia-ide/theia/issues"
+  },
+  "homepage": "https://github.com/theia-ide/theia",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "scripts": {
+    "prepare": "yarn run clean && yarn run build",
+    "clean": "theiaext clean",
+    "build": "theiaext build",
+    "watch": "theiaext watch",
+    "test": "theiaext test"
+  },
+  "nyc": {
+    "extends": "../../configs/nyc.json"
+  }
+}

packages/plugin-ext-metrics/src/browser/index.ts

@@ -0,0 +1,15 @@
+/********************************************************************************
+ * Copyright (C) 2019 Red Hat, Inc. and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/

packages/plugin-ext-metrics/src/browser/metric-output/plugin-metrics-success-output.ts

@@ -0,0 +1,31 @@
+/********************************************************************************
+ * Copyright (C) 2019 Red Hat, Inc. and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { MetricOutput, AnalyticsFromRequests } from '../plugin-metrics-interfaces';
+
+export class PluginMetricSuccessOutput implements MetricOutput {
+
+    public header = '# HELP language_server_success_metrics Percentage of successful language requests\n# TYPE language_server_success_metrics gauge\n';
+
+    createMetricOutput(id: string, method: string, requestAnalytics: AnalyticsFromRequests): string {
+        if (requestAnalytics.succesfulResponses < 0) {
+            requestAnalytics.succesfulResponses = 0;
+        }
+        const percentageOfSuccess = ((requestAnalytics.succesfulResponses / requestAnalytics.totalRequests) * 100);
+        return `language_server_success_metrics{id="${id}" method="${method}"} ${percentageOfSuccess}\n`;
+    }
+
+}

packages/plugin-ext-metrics/src/browser/metric-output/plugin-metrics-time-output.ts

@@ -0,0 +1,29 @@
+/********************************************************************************
+ * Copyright (C) 2019 Red Hat, Inc. and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { MetricOutput, AnalyticsFromRequests } from '../plugin-metrics-interfaces';
+
+export class PluginMetricTimeOutput implements MetricOutput {
+
+    public header =
+        '# HELP language_server_time_metrics Number of milliseconds it takes on average for a language server request\n# TYPE language_server_time_metrics gauge\n';
+
+    createMetricOutput(id: string, method: string, requestAnalytics: AnalyticsFromRequests): string {
+        const avgTime = requestAnalytics.avgTimeTaken;
+        return `language_server_time_metrics{id="${id}" method="${method}"} ${avgTime}\n`;
+    }
+
+}