From 088a4d57afa7a36c8f86c9afaa82d18477c82e5c Mon Sep 17 00:00:00 2001 From: Bartlomiej Obecny Date: Wed, 28 Oct 2020 20:06:07 +0100 Subject: [PATCH] Host Metrics (#227) --- examples/host-metrics/node.js | 22 ++ examples/host-metrics/package.json | 36 +++ package.json | 5 +- .../opentelemetry-host-metrics/.eslintignore | 1 + .../opentelemetry-host-metrics/.eslintrc.js | 7 + .../opentelemetry-host-metrics/.gitignore | 0 .../opentelemetry-host-metrics/.npmignore | 4 + packages/opentelemetry-host-metrics/LICENSE | 201 ++++++++++++ packages/opentelemetry-host-metrics/README.md | 61 ++++ .../opentelemetry-host-metrics/package.json | 69 +++++ .../opentelemetry-host-metrics/src/enum.ts | 38 +++ .../opentelemetry-host-metrics/src/index.ts | 18 ++ .../opentelemetry-host-metrics/src/metric.ts | 287 ++++++++++++++++++ .../src/stats/common.ts | 47 +++ .../src/stats/si.ts | 51 ++++ .../opentelemetry-host-metrics/src/types.ts | 55 ++++ .../opentelemetry-host-metrics/src/util.ts | 19 ++ .../opentelemetry-host-metrics/src/version.ts | 18 ++ .../test/metric.test.ts | 192 ++++++++++++ .../test/mocks/cpu.json | 4 + .../test/mocks/memory.json | 6 + .../test/mocks/network.json | 15 + .../opentelemetry-host-metrics/tsconfig.json | 12 + .../opentelemetry-host-metrics/tslint.json | 4 + 24 files changed, 1171 insertions(+), 1 deletion(-) create mode 100644 examples/host-metrics/node.js create mode 100644 examples/host-metrics/package.json create mode 100644 packages/opentelemetry-host-metrics/.eslintignore create mode 100644 packages/opentelemetry-host-metrics/.eslintrc.js create mode 100644 packages/opentelemetry-host-metrics/.gitignore create mode 100644 packages/opentelemetry-host-metrics/.npmignore create mode 100644 packages/opentelemetry-host-metrics/LICENSE create mode 100644 packages/opentelemetry-host-metrics/README.md create mode 100644 packages/opentelemetry-host-metrics/package.json create mode 100644 packages/opentelemetry-host-metrics/src/enum.ts create mode 100644 packages/opentelemetry-host-metrics/src/index.ts create mode 100644 packages/opentelemetry-host-metrics/src/metric.ts create mode 100644 packages/opentelemetry-host-metrics/src/stats/common.ts create mode 100644 packages/opentelemetry-host-metrics/src/stats/si.ts create mode 100644 packages/opentelemetry-host-metrics/src/types.ts create mode 100644 packages/opentelemetry-host-metrics/src/util.ts create mode 100644 packages/opentelemetry-host-metrics/src/version.ts create mode 100644 packages/opentelemetry-host-metrics/test/metric.test.ts create mode 100644 packages/opentelemetry-host-metrics/test/mocks/cpu.json create mode 100644 packages/opentelemetry-host-metrics/test/mocks/memory.json create mode 100644 packages/opentelemetry-host-metrics/test/mocks/network.json create mode 100644 packages/opentelemetry-host-metrics/tsconfig.json create mode 100644 packages/opentelemetry-host-metrics/tslint.json diff --git a/examples/host-metrics/node.js b/examples/host-metrics/node.js new file mode 100644 index 0000000000..682263578c --- /dev/null +++ b/examples/host-metrics/node.js @@ -0,0 +1,22 @@ +'use strict'; + +const { HostMetrics } = require('@opentelemetry/host-metrics'); +const { PrometheusExporter } = require('@opentelemetry/exporter-prometheus'); +const { MeterProvider } = require('@opentelemetry/metrics'); + +const exporter = new PrometheusExporter( + { + startServer: true, + }, + () => { + console.log('prometheus scrape endpoint: http://localhost:9464/metrics'); + }, +); + +const meterProvider = new MeterProvider({ + exporter, + interval: 2000, +}); + +const hostMetrics = new HostMetrics({ meterProvider, name: 'example-host-metrics' }); +hostMetrics.start(); diff --git a/examples/host-metrics/package.json b/examples/host-metrics/package.json new file mode 100644 index 0000000000..e7dad57377 --- /dev/null +++ b/examples/host-metrics/package.json @@ -0,0 +1,36 @@ +{ + "name": "host-metrics-example", + "private": true, + "version": "0.10.0", + "description": "Example of using @opentelemetry/host-metrics", + "main": "index.js", + "scripts": { + "start": "node node.js" + }, + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/open-telemetry/opentelemetry-js-contrib.git" + }, + "keywords": [ + "opentelemetry", + "http", + "tracing", + "metrics" + ], + "engines": { + "node": ">=8" + }, + "author": "OpenTelemetry Authors", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/open-telemetry/opentelemetry-js/issues" + }, + "dependencies": { + "@opentelemetry/api": "^0.12.0", + "@opentelemetry/core": "^0.12.0", + "@opentelemetry/exporter-prometheus": "^0.12.0", + "@opentelemetry/metrics": "^0.12.0", + "@opentelemetry/host-metrics": "^0.10.0" + }, + "homepage": "https://github.com/open-telemetry/opentelemetry-js#readme" +} diff --git a/package.json b/package.json index 59ca9f3b01..502e2e9051 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,9 @@ "internal": ":house: Internal", "documentation": ":memo: Documentation" }, - "ignoreCommitters": ["renovate-bot", "dependabot"] + "ignoreCommitters": [ + "renovate-bot", + "dependabot" + ] } } diff --git a/packages/opentelemetry-host-metrics/.eslintignore b/packages/opentelemetry-host-metrics/.eslintignore new file mode 100644 index 0000000000..378eac25d3 --- /dev/null +++ b/packages/opentelemetry-host-metrics/.eslintignore @@ -0,0 +1 @@ +build diff --git a/packages/opentelemetry-host-metrics/.eslintrc.js b/packages/opentelemetry-host-metrics/.eslintrc.js new file mode 100644 index 0000000000..f726f3becb --- /dev/null +++ b/packages/opentelemetry-host-metrics/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + "env": { + "mocha": true, + "node": true + }, + ...require('../../eslint.config.js') +} diff --git a/packages/opentelemetry-host-metrics/.gitignore b/packages/opentelemetry-host-metrics/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/opentelemetry-host-metrics/.npmignore b/packages/opentelemetry-host-metrics/.npmignore new file mode 100644 index 0000000000..9505ba9450 --- /dev/null +++ b/packages/opentelemetry-host-metrics/.npmignore @@ -0,0 +1,4 @@ +/bin +/coverage +/doc +/test diff --git a/packages/opentelemetry-host-metrics/LICENSE b/packages/opentelemetry-host-metrics/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/packages/opentelemetry-host-metrics/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/opentelemetry-host-metrics/README.md b/packages/opentelemetry-host-metrics/README.md new file mode 100644 index 0000000000..c4bc8fd4a2 --- /dev/null +++ b/packages/opentelemetry-host-metrics/README.md @@ -0,0 +1,61 @@ +#OpenTelemetry Host Metrics for Node.js +[![Gitter chat][gitter-image]][gitter-url] +[![NPM Published Version][npm-img]][npm-url] +[![dependencies][dependencies-image]][dependencies-url] +[![devDependencies][devDependencies-image]][devDependencies-url] +[![Apache License][license-image]][license-url] + +This module provides automatic collection of Host Metrics which includes metrics for: +* CPU +* Memory +* Network + +## Installation + +```bash +npm install --save @opentelemetry/host-metrics +``` + +## Usage + +```javascript +const { MeterProvider } = require('@opentelemetry/metrics'); +const { HostMetrics } = require('@opentelemetry/host-metrics'); +const { PrometheusExporter } = require('@opentelemetry/exporter-prometheus'); + +const exporter = new PrometheusExporter( + { startServer: true },() => { + console.log('prometheus scrape endpoint: http://localhost:9464/metrics'); + } +); + +const meterProvider = new MeterProvider({ + exporter, + interval: 2000, +}); + +const hostMetrics = new HostMetrics({ meterProvider, name: 'example-host-metrics' }); +hostMetrics.start(); + +``` + +## Useful links + +- For more information on OpenTelemetry, visit: +- For more about OpenTelemetry JavaScript: +- For help or feedback on this project, join us on [gitter][gitter-url] + +## License + +APACHE 2.0 - See [LICENSE][license-url] for more information. + +[gitter-image]: https://badges.gitter.im/open-telemetry/opentelemetry-js-contrib.svg +[gitter-url]: https://gitter.im/open-telemetry/opentelemetry-node?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +[license-url]: https://github.com/open-telemetry/opentelemetry-js-contrib/blob/master/LICENSE +[license-image]: https://img.shields.io/badge/license-Apache_2.0-green.svg?style=flat +[dependencies-image]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib.svg?path=packages%2Fopentelemetry-rca-metrics +[dependencies-url]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib?path=packages%2Fopentelemetry-rca-metrics +[devDependencies-image]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib.svg?path=packages%2Fopentelemetry-rca-metrics&type=dev +[devDependencies-url]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib?path=packages%2Fopentelemetry-rca-metrics&type=dev +[npm-url]: https://www.npmjs.com/package/@opentelemetry/rca-metrics +[npm-img]: https://badge.fury.io/js/%40opentelemetry%2Frca-metrics.svg diff --git a/packages/opentelemetry-host-metrics/package.json b/packages/opentelemetry-host-metrics/package.json new file mode 100644 index 0000000000..eaf5d5b4f8 --- /dev/null +++ b/packages/opentelemetry-host-metrics/package.json @@ -0,0 +1,69 @@ +{ + "name": "@opentelemetry/host-metrics", + "version": "0.10.0", + "description": "OpenTelemetry Host Metrics for Node.js", + "main": "build/src/index.js", + "types": "build/src/index.d.ts", + "repository": "open-telemetry/opentelemetry-js-contrib", + "scripts": { + "clean": "rimraf build/*", + "codecov": "nyc report --reporter=json && codecov -f coverage/*.json -p ../../", + "compile": "npm run version:update && tsc -p .", + "lint": "gts check", + "lint:fix": "gts fix", + "precompile": "tsc --version", + "prepare": "npm run compile", + "tdd": "npm run test -- --watch-extensions ts --watch", + "test": "nyc ts-mocha -p tsconfig.json test/**/*.test.ts", + "version:update": "node ../../scripts/version-update.js", + "watch": "tsc -w" + }, + "keywords": [ + "opentelemetry", + "metrics", + "nodejs", + "tracing", + "profiling", + "plugin" + ], + "author": "OpenTelemetry Authors", + "license": "Apache-2.0", + "engines": { + "node": ">=8.5.0" + }, + "files": [ + "build/src/**/*.js", + "build/src/**/*.d.ts", + "doc", + "LICENSE", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@opentelemetry/exporter-prometheus": "^0.12.0", + "@types/mocha": "8.0.2", + "@types/node": "14.0.27", + "@types/sinon": "9.0.4", + "codecov": "^3.7.2", + "gts": "^2.0.2", + "mocha": "7.2.0", + "mock-require": "^3.0.3", + "nan": "^2.14.1", + "nyc": "^15.1.0", + "rimraf": "^3.0.2", + "semver": "^7.3.2", + "sinon": "^9.0.3", + "ts-loader": "^8.0.3", + "ts-mocha": "^7.0.0", + "ts-node": "^8.10.2", + "typescript": "^3.9.7" + }, + "dependencies": { + "@opentelemetry/api": "^0.12.0", + "@opentelemetry/core": "^0.12.0", + "@opentelemetry/metrics": "^0.12.0", + "systeminformation": "^4.27.10" + } +} diff --git a/packages/opentelemetry-host-metrics/src/enum.ts b/packages/opentelemetry-host-metrics/src/enum.ts new file mode 100644 index 0000000000..8b55597156 --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/enum.ts @@ -0,0 +1,38 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export enum METRIC_NAMES { + CPU = 'cpu', + NETWORK = 'net', + MEMORY = 'mem', +} + +export enum CPU_LABELS { + USER = 'user', + SYSTEM = 'sys', + USAGE = 'usage', + TOTAL = 'total', +} + +export enum NETWORK_LABELS { + BYTES_SENT = 'bytesSent', + BYTES_RECEIVED = 'bytesRecv', +} + +export enum MEMORY_LABELS { + AVAILABLE = 'available', + TOTAL = 'total', +} diff --git a/packages/opentelemetry-host-metrics/src/index.ts b/packages/opentelemetry-host-metrics/src/index.ts new file mode 100644 index 0000000000..e0a66d91fe --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './metric'; +export * from './types'; diff --git a/packages/opentelemetry-host-metrics/src/metric.ts b/packages/opentelemetry-host-metrics/src/metric.ts new file mode 100644 index 0000000000..ba6ae853ef --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/metric.ts @@ -0,0 +1,287 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as api from '@opentelemetry/api'; +import * as metrics from '@opentelemetry/metrics'; +import * as enums from './enum'; + +import { getCpuUsageData, getMemoryData } from './stats/common'; +import { getNetworkData } from './stats/si'; + +import * as types from './types'; +import { VERSION } from './version'; + +/** + * Metrics Collector Configuration + */ +export interface MetricsCollectorConfig { + logger?: api.Logger; + // maximum timeout to wait for stats collection default is 500ms + maxTimeoutUpdateMS?: number; + // Meter Provider + meterProvider: metrics.MeterProvider; + // Character to be used to join metrics - default is "." + metricNameSeparator?: string; + // Name of component + name: string; + // metric export endpoint + url: string; +} + +const DEFAULT_MAX_TIMEOUT_UPDATE_MS = 500; +const DEFAULT_NAME = 'opentelemetry-host-metrics'; +const DEFAULT_METRIC_NAME_SEPARATOR = '.'; + +// default label name to be used to store metric name +const DEFAULT_KEY = 'name'; + +/** + * Metrics Collector - collects metrics for CPU, Memory, Heap, Network, Event + * Loop, Garbage Collector, Heap Space + * the default label name for metric name is "name" + */ +export class HostMetrics { + protected _logger: api.Logger | undefined; + protected _maxTimeoutUpdateMS: number; + protected _meter: metrics.Meter; + private _name: string; + private _boundCounters: { [key: string]: api.BoundCounter } = {}; + private _metricNameSeparator: string; + + private _memValueObserver: types.ValueObserverWithObservations | undefined; + + constructor(config: MetricsCollectorConfig) { + this._logger = config.logger || new api.NoopLogger(); + this._name = config.name || DEFAULT_NAME; + this._maxTimeoutUpdateMS = + config.maxTimeoutUpdateMS || DEFAULT_MAX_TIMEOUT_UPDATE_MS; + this._metricNameSeparator = + config.metricNameSeparator || DEFAULT_METRIC_NAME_SEPARATOR; + const meterProvider = + config.meterProvider || api.metrics.getMeterProvider(); + if (!config.meterProvider) { + this._logger.warn('No meter provider, using default'); + } + this._meter = meterProvider.getMeter(this._name, VERSION); + } + + /** + * Creates a metric key name based on metric name and a key + * @param metricName + * @param key + */ + protected _boundKey(metricName: string, key: string) { + if (!key) { + return metricName; + } + return `${metricName}${this._metricNameSeparator}${key}`; + } + + /** + * Updates counter based on boundkey + * @param metricName + * @param key + * @param value + */ + protected _counterUpdate(metricName: string, key: string, value = 0) { + const boundKey = this._boundKey(metricName, key); + this._boundCounters[boundKey].add(value); + } + + /** + * @param metricName metric name - this will be added as label under name + * "name" + * @param values values to be used to generate bound counters for each + * value prefixed with metricName + * @param description metric description + */ + protected _createCounter( + metricName: string, + values: string[], + description?: string + ) { + const keys = values.map(key => this._boundKey(metricName, key)); + const counter = this._meter.createCounter(metricName, { + description: description || metricName, + }); + keys.forEach(key => { + this._boundCounters[key] = counter.bind({ [DEFAULT_KEY]: key }); + }); + } + + /** + * @param metricName metric name - this will be added as label under name + * "name" + * @param values values to be used to generate full metric name + * (metricName + value) + * value prefixed with metricName + * @param description metric description + * @param labelKey extra label to be observed + * @param labelValues label values to be observed based on labelKey + * @param afterKey extra name to be added to full metric name + */ + protected _createValueObserver( + metricName: string, + values: string[], + description: string, + labelKey = '', + labelValues: string[] = [], + afterKey = '' + ): types.ValueObserverWithObservations { + const labelKeys = [DEFAULT_KEY]; + if (labelKey) { + labelKeys.push(labelKey); + } + const observer = this._meter.createValueObserver(metricName, { + description: description || metricName, + }); + + const observations: types.Observations[] = []; + values.forEach(value => { + const boundKey = this._boundKey( + this._boundKey(metricName, value), + afterKey + ); + if (labelKey) { + // there is extra label to be observed mixed with default one + // for example we want to be able to observe "name" and "gc_type" + labelValues.forEach(label => { + const observedLabels = Object.assign( + {}, + { [DEFAULT_KEY]: boundKey }, + { + [labelKey]: label, + } + ); + observations.push({ + key: value, + labels: observedLabels, + labelKey, + }); + }); + } else { + observations.push({ + key: value, + labels: { [DEFAULT_KEY]: boundKey }, + }); + } + }); + + return { observer, observations }; + } + + // MEMORY + private _createMemValueObserver() { + this._memValueObserver = this._createValueObserver( + enums.METRIC_NAMES.MEMORY, + Object.values(enums.MEMORY_LABELS), + 'Memory' + ); + } + + /** + * Updates observer + * @param observerBatchResult + * @param data + * @param observerWithObservations + */ + protected _updateObserver( + observerBatchResult: api.BatchObserverResult, + data: DataType, + observerWithObservations?: types.ValueObserverWithObservations + ) { + if (observerWithObservations) { + observerWithObservations.observations.forEach(observation => { + const value = data[observation.key as keyof DataType]; + if (typeof value === 'number') { + observerBatchResult.observe(observation.labels, [ + observerWithObservations.observer.observation(value), + ]); + } + }); + } + } + + /** + * Creates metrics + */ + protected _createMetrics() { + // CPU COUNTER + this._createCounter( + enums.METRIC_NAMES.CPU, + Object.values(enums.CPU_LABELS), + 'CPU Usage' + ); + + // NETWORK COUNTER + this._createCounter( + enums.METRIC_NAMES.NETWORK, + Object.values(enums.NETWORK_LABELS), + 'Network Usage' + ); + + // MEMORY + this._createMemValueObserver(); + + this._meter.createBatchObserver( + 'metric_batch_observer', + observerBatchResult => { + Promise.all([ + getMemoryData(), + getCpuUsageData(), + getNetworkData(), + ]).then(([memoryData, cpuUsage, networkData]) => { + // CPU COUNTER + Object.values(enums.CPU_LABELS).forEach(value => { + this._counterUpdate(enums.METRIC_NAMES.CPU, value, cpuUsage[value]); + }); + + // NETWORK COUNTER + Object.values(enums.NETWORK_LABELS).forEach(value => { + this._counterUpdate( + enums.METRIC_NAMES.NETWORK, + value, + networkData[value] + ); + }); + + // MEMORY + this._updateObserver( + observerBatchResult, + memoryData, + this._memValueObserver + ); + }); + }, + { + maxTimeoutUpdateMS: this._maxTimeoutUpdateMS, + logger: this._logger, + } + ); + } + + /** + * Starts collecting stats + */ + start() { + // initial collection + Promise.all([getMemoryData(), getCpuUsageData(), getNetworkData()]).then( + () => { + this._createMetrics(); + } + ); + } +} diff --git a/packages/opentelemetry-host-metrics/src/stats/common.ts b/packages/opentelemetry-host-metrics/src/stats/common.ts new file mode 100644 index 0000000000..d3c25d1c4a --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/stats/common.ts @@ -0,0 +1,47 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as os from 'os'; +import { CPU_LABELS, MEMORY_LABELS } from '../enum'; + +import { CpuUsageData, MemoryData } from '../types'; + +const MICROSECOND = 1 / 1e6; +let cpuUsage: NodeJS.CpuUsage | undefined; + +/** + * It returns cpu load delta from last time + */ +export function getCpuUsageData(): CpuUsageData { + const elapsedUsage = process.cpuUsage(cpuUsage); + cpuUsage = process.cpuUsage(); + return { + [CPU_LABELS.USER]: elapsedUsage.user * MICROSECOND, + [CPU_LABELS.SYSTEM]: elapsedUsage.system * MICROSECOND, + [CPU_LABELS.USAGE]: (elapsedUsage.user + elapsedUsage.system) * MICROSECOND, + [CPU_LABELS.TOTAL]: (cpuUsage.user + cpuUsage.system) * MICROSECOND, + }; +} + +/** + * Returns memory data stats + */ +export function getMemoryData(): MemoryData { + return { + [MEMORY_LABELS.AVAILABLE]: os.freemem(), + [MEMORY_LABELS.TOTAL]: os.totalmem(), + }; +} diff --git a/packages/opentelemetry-host-metrics/src/stats/si.ts b/packages/opentelemetry-host-metrics/src/stats/si.ts new file mode 100644 index 0000000000..4fb8c1f292 --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/stats/si.ts @@ -0,0 +1,51 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as SI from 'systeminformation'; +import { NetworkData } from '../types'; +import { ObjectKeys } from '../util'; + +let previousNetworkStats: Partial = {}; + +/** + * It returns network usage delta from last time + */ +export function getNetworkData() { + return new Promise(resolve => { + const stats: NetworkData = { + bytesRecv: 0, + bytesSent: 0, + }; + SI.networkStats() + .then(results => { + results.forEach(result => { + stats.bytesRecv += result.rx_bytes; + stats.bytesSent += result.tx_bytes; + }); + const lastStats = Object.assign({}, stats); + + ObjectKeys(stats).forEach(key => { + stats[key] = stats[key] - (previousNetworkStats[key] || 0); + }); + + previousNetworkStats = lastStats; + resolve(stats); + }) + .catch(() => { + resolve(stats); + }); + }); +} diff --git a/packages/opentelemetry-host-metrics/src/types.ts b/packages/opentelemetry-host-metrics/src/types.ts new file mode 100644 index 0000000000..b47f4ed4a4 --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/types.ts @@ -0,0 +1,55 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { ValueObserver } from '@opentelemetry/api'; + +/** + * Network data + */ +export interface NetworkData { + // bytes received + bytesRecv: number; + // bytes sent + bytesSent: number; +} + +/** + * CPU usage data + */ +export interface CpuUsageData { + sys: number; + usage: number; + user: number; + total: number; +} + +/** + * Memory data + */ +export interface MemoryData { + available: number; + total: number; +} + +export interface Observations { + key: string; + labels: Record; + labelKey?: string; +} + +export interface ValueObserverWithObservations { + observer: ValueObserver; + observations: Observations[]; +} diff --git a/packages/opentelemetry-host-metrics/src/util.ts b/packages/opentelemetry-host-metrics/src/util.ts new file mode 100644 index 0000000000..e26b0f7610 --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/util.ts @@ -0,0 +1,19 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function ObjectKeys(t: T) { + return Object.keys(t) as (keyof T)[]; +} diff --git a/packages/opentelemetry-host-metrics/src/version.ts b/packages/opentelemetry-host-metrics/src/version.ts new file mode 100644 index 0000000000..aa9b19601b --- /dev/null +++ b/packages/opentelemetry-host-metrics/src/version.ts @@ -0,0 +1,18 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// this is autogenerated file, see scripts/version-update.js +export const VERSION = '0.10.0'; diff --git a/packages/opentelemetry-host-metrics/test/metric.test.ts b/packages/opentelemetry-host-metrics/test/metric.test.ts new file mode 100644 index 0000000000..b152a31a29 --- /dev/null +++ b/packages/opentelemetry-host-metrics/test/metric.test.ts @@ -0,0 +1,192 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const SI = require('systeminformation'); +import { ExportResult } from '@opentelemetry/core'; +import { + Histogram, + MeterProvider, + MetricExporter, + MetricRecord, +} from '@opentelemetry/metrics'; +import * as assert from 'assert'; +import * as os from 'os'; +import * as sinon from 'sinon'; + +const cpuJson = require('./mocks/cpu.json'); +const networkJson = require('./mocks/network.json'); +const memoryJson = require('./mocks/memory.json'); + +class NoopExporter implements MetricExporter { + export( + metrics: MetricRecord[], + resultCallback: (result: ExportResult) => void + ): void {} + + shutdown(): Promise { + return Promise.resolve(); + } +} + +const originalSetTimeout = setTimeout; + +let countSI = 0; +const mockedSI = { + networkStats: function () { + return new Promise((resolve, reject) => { + countSI++; + const stats: any[] = networkJson + .slice() + .map((obj: any) => Object.assign({}, obj)); + + for (let i = 0, j = networkJson.length; i < j; i++) { + Object.keys(stats[i]).forEach(key => { + if (typeof stats[i][key] === 'number' && stats[i][key] > 0) { + stats[i][key] = stats[i][key] * countSI; + } + }); + } + resolve(stats); + }); + }, +}; + +const mockedOS = { + freemem: function () { + return 7179869184; + }, + totalmem: function () { + return 17179869184; + }, +}; + +const INTERVAL = 3000; + +let metrics: any; + +describe('Host Metrics', () => { + let sandbox: sinon.SinonSandbox; + let hostMetrics: any; + let exporter: MetricExporter; + let exportSpy: any; + + beforeEach(done => { + sandbox = sinon.createSandbox(); + sandbox.useFakeTimers(); + + sandbox.stub(os, 'freemem').returns(mockedOS.freemem()); + sandbox.stub(os, 'totalmem').returns(mockedOS.totalmem()); + sandbox.stub(process, 'cpuUsage').returns(cpuJson); + sandbox.stub(process, 'memoryUsage').returns(memoryJson); + const spyNetworkStats = sandbox + .stub(SI, 'networkStats') + .returns(mockedSI.networkStats()); + + exporter = new NoopExporter(); + exportSpy = sandbox.stub(exporter, 'export'); + + const meterProvider = new MeterProvider({ + interval: INTERVAL, + exporter, + }); + + // it seems like this is the only way to be able to mock + // `node-gyp-build` before metrics are being loaded, if import them before + // the first pass on unit tests will not mock correctly + metrics = require('../src'); + hostMetrics = new metrics.HostMetrics({ + meterProvider, + name: 'opentelemetry-host-metrics', + }); + hostMetrics.start(); + + // because networkStats mock simulates the network with every call it + // returns the data that is bigger then previous, it needs to stub it again + // as network is also called in initial start to start counting from 0 + spyNetworkStats.restore(); + sandbox.stub(SI, 'networkStats').returns(mockedSI.networkStats()); + + // sinon fake doesn't work fine with setImmediate + originalSetTimeout(() => { + // move the clock with the same value as interval + sandbox.clock.tick(INTERVAL); + // move to "real" next tick so that async batcher observer will start + // processing metrics + originalSetTimeout(() => { + // allow all calbacks to finish correctly as they are finishing in + // next tick due to async + sandbox.clock.tick(1); + originalSetTimeout(() => { + done(); + }); + }); + }); + }); + afterEach(() => { + sandbox.restore(); + }); + + it('should create a new instance', () => { + assert.ok(hostMetrics instanceof metrics.HostMetrics); + }); + + it('should create a new instance with default meter provider', () => { + hostMetrics = new metrics.HostMetrics({ + name: 'opentelemetry-host-metrics', + }); + hostMetrics.start(); + assert.ok(hostMetrics instanceof metrics.HostMetrics); + }); + + it('should export CPU metrics', () => { + const records = getRecords(exportSpy.args[0][0], 'cpu'); + assert.strictEqual(records.length, 4); + ensureValue(records[0], 'cpu.user', 1.899243); + ensureValue(records[1], 'cpu.sys', 0.258553); + ensureValue(records[2], 'cpu.usage', 2.157796); + ensureValue(records[3], 'cpu.total', 2.157796); + }); + + it('should export Network metrics', done => { + const records = getRecords(exportSpy.args[0][0], 'net'); + assert.strictEqual(records.length, 2); + ensureValue(records[0], 'net.bytesSent', 14207163202); + ensureValue(records[1], 'net.bytesRecv', 60073930753); + done(); + }); + + it('should export Memory metrics', done => { + const records = getRecords(exportSpy.args[0][0], 'mem'); + assert.strictEqual(records.length, 2); + ensureValue(records[0], 'mem.available', mockedOS.freemem()); + ensureValue(records[1], 'mem.total', mockedOS.totalmem()); + done(); + }); +}); + +function getRecords(records: MetricRecord[], name: string): MetricRecord[] { + return records.filter(record => record.descriptor.name === name); +} + +function ensureValue(record: MetricRecord, name: string, value: number) { + assert.strictEqual(record.labels.name, name); + const point = record.aggregator.toPoint(); + const aggValue = + typeof point.value === 'number' + ? point.value + : (point.value as Histogram).sum; + assert.strictEqual(aggValue, value); +} diff --git a/packages/opentelemetry-host-metrics/test/mocks/cpu.json b/packages/opentelemetry-host-metrics/test/mocks/cpu.json new file mode 100644 index 0000000000..ee4ae3bf1f --- /dev/null +++ b/packages/opentelemetry-host-metrics/test/mocks/cpu.json @@ -0,0 +1,4 @@ +{ + "user": 1899243, + "system": 258553 +} \ No newline at end of file diff --git a/packages/opentelemetry-host-metrics/test/mocks/memory.json b/packages/opentelemetry-host-metrics/test/mocks/memory.json new file mode 100644 index 0000000000..2a565068ca --- /dev/null +++ b/packages/opentelemetry-host-metrics/test/mocks/memory.json @@ -0,0 +1,6 @@ +{ + "rss": 152535045, + "heapTotal": 82182145, + "heapUsed": 53736445, + "external": 1543181 +} \ No newline at end of file diff --git a/packages/opentelemetry-host-metrics/test/mocks/network.json b/packages/opentelemetry-host-metrics/test/mocks/network.json new file mode 100644 index 0000000000..77caed7d72 --- /dev/null +++ b/packages/opentelemetry-host-metrics/test/mocks/network.json @@ -0,0 +1,15 @@ +[ + { + "iface": "en0", + "operstate": "up", + "rx_bytes": 60073930753, + "rx_dropped": 1200, + "rx_errors": 0, + "tx_bytes": 14207163202, + "tx_dropped": 1200, + "tx_errors": 21104, + "rx_sec": -1, + "tx_sec": -1, + "ms": 0 + } +] \ No newline at end of file diff --git a/packages/opentelemetry-host-metrics/tsconfig.json b/packages/opentelemetry-host-metrics/tsconfig.json new file mode 100644 index 0000000000..4b1645af66 --- /dev/null +++ b/packages/opentelemetry-host-metrics/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base", + "compilerOptions": { + "rootDir": ".", + "outDir": "build" + }, + "include": [ + "src/**/*.ts", + "scripts/**/*.js", + "test/**/*.ts" + ] +} diff --git a/packages/opentelemetry-host-metrics/tslint.json b/packages/opentelemetry-host-metrics/tslint.json new file mode 100644 index 0000000000..0710b135d0 --- /dev/null +++ b/packages/opentelemetry-host-metrics/tslint.json @@ -0,0 +1,4 @@ +{ + "rulesDirectory": ["node_modules/tslint-microsoft-contrib"], + "extends": ["../../tslint.base.js", "./node_modules/tslint-consistent-codestyle"] +}