From e7b2d461a62c771cedb34383d43a42510a735f5c Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Mon, 12 Feb 2024 14:02:15 -0800 Subject: [PATCH] chore: remove mmap-based CircularBuffer --- packages/telemetry/package.json | 3 +- packages/telemetry/src/flight-recorder.js | 96 ------------------- packages/telemetry/src/frcat-entrypoint.js | 4 +- .../telemetry/test/test-flight-recorder.js | 5 +- patches/bufferfromfile+0.4.377.patch | 9 -- yarn.lock | 78 ++------------- 6 files changed, 10 insertions(+), 185 deletions(-) delete mode 100644 patches/bufferfromfile+0.4.377.patch diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index 77653297e16b..352ea489cbf9 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -37,7 +37,6 @@ "@opentelemetry/semantic-conventions": "~1.9.0", "anylogger": "^0.21.0", "better-sqlite3": "^9.1.1", - "bufferfromfile": "agoric-labs/BufferFromFile#Agoric-built", "tmp": "^0.2.1" }, "devDependencies": { @@ -64,6 +63,6 @@ "workerThreads": false }, "typeCoverage": { - "atLeast": 87.55 + "atLeast": 87.14 } } diff --git a/packages/telemetry/src/flight-recorder.js b/packages/telemetry/src/flight-recorder.js index 59847c6cbfc4..cd7f795a5b2e 100644 --- a/packages/telemetry/src/flight-recorder.js +++ b/packages/telemetry/src/flight-recorder.js @@ -2,19 +2,6 @@ /* global Buffer */ /// -// https://github.com/Agoric/agoric-sdk/issues/3742#issuecomment-1028451575 -// I'd mmap() a 100MB file, reserve a few bytes for offsets, then use the rest -// as a circular buffer to hold length-prefixed records. The agd process would -// keep writing new events into the RAM window and updating the start/end -// pointers, with some sequencing to make sure the record gets written before -// the pointer is updated. Then, no mattter how abruptly the process is -// terminated, as long as the host computer itself is still running, the on-disk -// file would contain the most recent state, and anybody who reads the file will -// get the most recent state. The host kernel (linux) is under no obligation to -// flush it to disk any particular time, but knows when reads happen, so there's -// no coherency problem, and the speed is unaffected by disk write speeds. - -import BufferFromFile from 'bufferfromfile'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; import path from 'node:path'; @@ -192,8 +179,6 @@ function finishCircularBuffer(arenaSize, header, readRecord, writeRecord) { } /** - * Variant of makeMemoryMappedCircularBuffer that writes to a file instead of using BufferFromFile - * * @param {{ * circularBufferSize?: number, * stateDir?: string, @@ -283,87 +268,6 @@ export const makeSimpleCircularBuffer = async ({ return finishCircularBuffer(arenaSize, header, readRecord, writeRecord); }; -/** - * @param {{ - * circularBufferSize?: number, - * stateDir?: string, - * circularBufferFilename?: string - * }} opts - */ -export const makeMemoryMappedCircularBuffer = async ({ - circularBufferSize = DEFAULT_CBUF_SIZE, - stateDir = '/tmp', - circularBufferFilename, -}) => { - const filename = circularBufferFilename || `${stateDir}/${DEFAULT_CBUF_FILE}`; - - const newArenaSize = await initializeCircularBuffer( - filename, - circularBufferSize, - ); - - /** - * @type {Uint8Array} - * BufferFromFile mmap()s the file into the process address space. - */ - const fileBuf = BufferFromFile(filename).Uint8Array(); - const header = new DataView(fileBuf.buffer, 0, I_ARENA_START); - - // Detect the arena size from the header, if not initialized. - const hdrArenaSize = header.getBigUint64(I_ARENA_SIZE); - const arenaSize = newArenaSize || hdrArenaSize; - - const hdrMagic = header.getBigUint64(I_MAGIC); - SLOG_MAGIC === hdrMagic || - Fail`${filename} is not a slog buffer; wanted magic ${SLOG_MAGIC}, got ${hdrMagic}`; - arenaSize === hdrArenaSize || - Fail`${filename} arena size mismatch; wanted ${arenaSize}, got ${hdrArenaSize}`; - const arena = new Uint8Array( - fileBuf.buffer, - header.byteLength, - Number(arenaSize), - ); - - /** - * - * @param {Uint8Array} outbuf - * @param {number} readStart - * @param {number} firstReadLength - */ - - function readRecord(outbuf, readStart, firstReadLength) { - outbuf.set(arena.subarray(readStart, readStart + firstReadLength)); - if (firstReadLength < outbuf.byteLength) { - outbuf.set( - arena.subarray(0, outbuf.byteLength - firstReadLength), - firstReadLength, - ); - } - } - - /** - * - * @param {Uint8Array} record - * @param {number} firstWriteLength - * @param {bigint} circEnd - */ - const writeRecord = (record, firstWriteLength, circEnd) => { - arena.set(record.subarray(0, firstWriteLength), Number(circEnd)); - if (firstWriteLength < record.byteLength) { - // Write to the beginning of the arena. - arena.set(record.subarray(firstWriteLength, record.byteLength), 0); - } - return Promise.resolve(); - }; - - /** - * @param {Uint8Array} outbuf - * @param {number} [offset] offset relative to the current trailing edge (circStart) of the data - * @returns {IteratorResult} - */ - return finishCircularBuffer(arenaSize, header, readRecord, writeRecord); -}; - /** * * @param {Pick>, 'writeCircBuf'>} circBuf diff --git a/packages/telemetry/src/frcat-entrypoint.js b/packages/telemetry/src/frcat-entrypoint.js index 63dfc264bca7..34b8f0420123 100755 --- a/packages/telemetry/src/frcat-entrypoint.js +++ b/packages/telemetry/src/frcat-entrypoint.js @@ -5,7 +5,7 @@ import '@endo/init'; -import { makeMemoryMappedCircularBuffer } from './flight-recorder.js'; +import { makeSimpleCircularBuffer } from './flight-recorder.js'; const main = async () => { const files = process.argv.slice(2); @@ -14,7 +14,7 @@ const main = async () => { } for await (const file of files) { - const { readCircBuf } = await makeMemoryMappedCircularBuffer({ + const { readCircBuf } = await makeSimpleCircularBuffer({ circularBufferFilename: file, circularBufferSize: 0, }); diff --git a/packages/telemetry/test/test-flight-recorder.js b/packages/telemetry/test/test-flight-recorder.js index 69a6705bccdf..27cfada2c3a1 100644 --- a/packages/telemetry/test/test-flight-recorder.js +++ b/packages/telemetry/test/test-flight-recorder.js @@ -3,11 +3,11 @@ import tmp from 'tmp'; import { test } from './prepare-test-env-ava.js'; import { - makeMemoryMappedCircularBuffer, makeSimpleCircularBuffer, makeSlogSenderFromBuffer, } from '../src/flight-recorder.js'; +// Factored this way to support multiple implementations, which at one point there were const bufferTests = test.macro( /** * @@ -78,9 +78,6 @@ const bufferTests = test.macro( }, ); -test('memory mapped', bufferTests, { - makeBuffer: makeMemoryMappedCircularBuffer, -}); test('simple', bufferTests, { makeBuffer: makeSimpleCircularBuffer, }); diff --git a/patches/bufferfromfile+0.4.377.patch b/patches/bufferfromfile+0.4.377.patch deleted file mode 100644 index 51690d3cc728..000000000000 --- a/patches/bufferfromfile+0.4.377.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff --git a/node_modules/bufferfromfile/proto/wtools/amid/bufferFromFile/Main.js b/node_modules/bufferfromfile/proto/wtools/amid/bufferFromFile/Main.js -index 4c4c41b..9b88d11 100644 ---- a/node_modules/bufferfromfile/proto/wtools/amid/bufferFromFile/Main.js -+++ b/node_modules/bufferfromfile/proto/wtools/amid/bufferFromFile/Main.js -@@ -1,3 +1,4 @@ -+// @ts-nocheck - ( function _BufferFromFile_js_() - { // - diff --git a/yarn.lock b/yarn.lock index 8679affb8bf5..a40ad1e043b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1624,21 +1624,6 @@ npmlog "^6.0.2" write-file-atomic "^4.0.1" -"@mapbox/node-pre-gyp@1.0.9": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.9.tgz#09a8781a3a036151cdebbe8719d6f8b25d4058bc" - integrity sha512-aDF3S3rK9Q2gey/WAttUlISduDItz5BU3306M9Eyv6/oS40aMprnopshtlKTykxRNIBEZuRMaZAnbrQ4QtKGyw== - dependencies: - detect-libc "^2.0.0" - https-proxy-agent "^5.0.0" - make-dir "^3.1.0" - node-fetch "^2.6.7" - nopt "^5.0.0" - npmlog "^5.0.1" - rimraf "^3.0.2" - semver "^7.3.5" - tar "^6.1.11" - "@noble/hashes@^1", "@noble/hashes@^1.0.0": version "1.1.2" resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.1.2.tgz#e9e035b9b166ca0af657a7848eb2718f0f22f183" @@ -2883,14 +2868,6 @@ are-docs-informative@^0.0.2: resolved "https://registry.yarnpkg.com/are-docs-informative/-/are-docs-informative-0.0.2.tgz#387f0e93f5d45280373d387a59d34c96db321963" integrity sha512-ixiS0nLNNG5jNQzgZJNoUpBKdo9yTYZMGJ+QgT2jmjR7G7+QHRCc4v6LQ3NgE7EBJq+o0ams3waJwkrlBom8Ig== -are-we-there-yet@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz#372e0e7bd279d8e94c653aaa1f67200884bf3e1c" - integrity sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw== - dependencies: - delegates "^1.0.0" - readable-stream "^3.6.0" - are-we-there-yet@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz#679df222b278c64f2cdba1175cdc00b0d96164bd" @@ -3299,14 +3276,6 @@ buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" -bufferfromfile@agoric-labs/BufferFromFile#Agoric-built: - version "0.4.377" - resolved "https://codeload.github.com/agoric-labs/BufferFromFile/tar.gz/691031035856fadba2603e52f4411160d7f34ed6" - dependencies: - "@mapbox/node-pre-gyp" "1.0.9" - node-gyp "^9.3.1" - wbasenodejscpp latest - builtin-modules@^3.1.0, builtin-modules@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.3.0.tgz#cae62812b89801e9656336e46223e030386be7b6" @@ -3647,7 +3616,7 @@ color-string@^1.6.0: color-name "^1.0.0" simple-swizzle "^0.2.2" -color-support@^1.1.2, color-support@^1.1.3: +color-support@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== @@ -3768,7 +3737,7 @@ confusing-browser-globals@^1.0.10: resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81" integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA== -console-control-strings@^1.0.0, console-control-strings@^1.1.0: +console-control-strings@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= @@ -5253,21 +5222,6 @@ functions-have-names@^1.2.2: resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.2.tgz#98d93991c39da9361f8e50b337c4f6e41f120e21" integrity sha512-bLgc3asbWdwPbx2mNk2S49kmJCuQeu0nfmaOgbs8WIyzzkw3r4htszdIi9Q9EMezDPTYuJx2wvjZ/EwgAthpnA== -gauge@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-3.0.2.tgz#03bf4441c044383908bcfa0656ad91803259b395" - integrity sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q== - dependencies: - aproba "^1.0.3 || ^2.0.0" - color-support "^1.1.2" - console-control-strings "^1.0.0" - has-unicode "^2.0.1" - object-assign "^4.1.1" - signal-exit "^3.0.0" - string-width "^4.2.3" - strip-ansi "^6.0.1" - wide-align "^1.1.2" - gauge@^4.0.3: version "4.0.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-4.0.4.tgz#52ff0652f2bbf607a989793d53b751bef2328dce" @@ -6681,7 +6635,7 @@ make-dir@^2.1.0: pify "^4.0.1" semver "^5.6.0" -make-dir@^3.0.0, make-dir@^3.1.0: +make-dir@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== @@ -7422,7 +7376,7 @@ node-gyp-build@^4.3.0, node-gyp-build@^4.4.0, node-gyp-build@^4.5.0: resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.0.tgz#0c52e4cbf54bbd28b709820ef7b6a3c2d6209055" integrity sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ== -node-gyp@^9.0.0, node-gyp@^9.3.1: +node-gyp@^9.0.0: version "9.4.0" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.0.tgz#2a7a91c7cba4eccfd95e949369f27c9ba704f369" integrity sha512-dMXsYP6gc9rRbejLXmTbVRYjAHw7ppswsKyMxuxJxxOHzluIO1rGp9TOQgjFJ+2MCqcOcQTOPB/8Xwhr+7s4Eg== @@ -7610,16 +7564,6 @@ npm-run-path@^5.1.0: dependencies: path-key "^4.0.0" -npmlog@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-5.0.1.tgz#f06678e80e29419ad67ab964e0fa69959c1eb8b0" - integrity sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw== - dependencies: - are-we-there-yet "^2.0.0" - console-control-strings "^1.1.0" - gauge "^3.0.0" - set-blocking "^2.0.0" - npmlog@^6.0.0, npmlog@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-6.0.2.tgz#c8166017a42f2dea92d6453168dd865186a70830" @@ -7681,11 +7625,6 @@ nx@15.9.4, "nx@>=14.8.1 < 16": "@nrwl/nx-win32-arm64-msvc" "15.9.4" "@nrwl/nx-win32-x64-msvc" "15.9.4" -object-assign@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= - object-hash@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df" @@ -8937,7 +8876,7 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: +signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== @@ -9877,11 +9816,6 @@ walk-up-path@^1.0.0: resolved "https://registry.yarnpkg.com/walk-up-path/-/walk-up-path-1.0.0.tgz#d4745e893dd5fd0dbb58dd0a4c6a33d9c9fec53e" integrity sha512-hwj/qMDUEjCU5h0xr90KGCf0tg0/LgJbmOWgrWKYlcJZM7XvquvUJZ0G/HMGr7F7OQMOUuPHWP9JpriinkAlkg== -wbasenodejscpp@latest: - version "0.3.134" - resolved "https://registry.yarnpkg.com/wbasenodejscpp/-/wbasenodejscpp-0.3.134.tgz#c23c06bd8c3e170afe59084cfa6e7d41c07b0ffa" - integrity sha512-B3cyJ14XNKe7gfu+pQPdCkDKTMu4DiFS/iKWE+1AhroChj+hLiNYCcoUuvsjEocQtBPItjb57mbhdKCJ80pjCA== - wcwidth@^1.0.0, wcwidth@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8" @@ -9944,7 +9878,7 @@ which@^2.0.1, which@^2.0.2: dependencies: isexe "^2.0.0" -wide-align@^1.1.2, wide-align@^1.1.5: +wide-align@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.5.tgz#df1d4c206854369ecf3c9a4898f1b23fbd9d15d3" integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==