Skip to content

Commit

Permalink
chore: remove mmap-based CircularBuffer
Browse files Browse the repository at this point in the history
  • Loading branch information
@turadg
turadg committed Feb 14, 2024
1 parent 47a2add commit 515ef12
Show file tree
Hide file tree
Showing 6 changed files with 10 additions and 185 deletions.
3 changes: 1 addition & 2 deletions packages/telemetry/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
"@opentelemetry/semantic-conventions": "~1.9.0",
"anylogger": "^0.21.0",
"better-sqlite3": "^9.1.1",
"bufferfromfile": "agoric-labs/BufferFromFile#Agoric-built",
"tmp": "^0.2.1"
},
"devDependencies": {
Expand All @@ -64,6 +63,6 @@
"workerThreads": false
},
"typeCoverage": {
"atLeast": 87.55
"atLeast": 87.14
}
}
96 changes: 0 additions & 96 deletions packages/telemetry/src/flight-recorder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,6 @@
/* global Buffer */
/// <reference types="ses" />

// https://github.com/Agoric/agoric-sdk/issues/3742#issuecomment-1028451575
// I'd mmap() a 100MB file, reserve a few bytes for offsets, then use the rest
// as a circular buffer to hold length-prefixed records. The agd process would
// keep writing new events into the RAM window and updating the start/end
// pointers, with some sequencing to make sure the record gets written before
// the pointer is updated. Then, no mattter how abruptly the process is
// terminated, as long as the host computer itself is still running, the on-disk
// file would contain the most recent state, and anybody who reads the file will
// get the most recent state. The host kernel (linux) is under no obligation to
// flush it to disk any particular time, but knows when reads happen, so there's
// no coherency problem, and the speed is unaffected by disk write speeds.

import BufferFromFile from 'bufferfromfile';
import fs from 'node:fs';
import fsp from 'node:fs/promises';
import path from 'node:path';
Expand Down Expand Up @@ -192,8 +179,6 @@ function finishCircularBuffer(arenaSize, header, readRecord, writeRecord) {
}

/**
* Variant of makeMemoryMappedCircularBuffer that writes to a file instead of using BufferFromFile
*
* @param {{
* circularBufferSize?: number,
* stateDir?: string,
Expand Down Expand Up @@ -284,87 +269,6 @@ export const makeSimpleCircularBuffer = async ({
return finishCircularBuffer(arenaSize, header, readRecord, writeRecord);
};

/**
* @param {{
* circularBufferSize?: number,
* stateDir?: string,
* circularBufferFilename?: string
* }} opts
*/
export const makeMemoryMappedCircularBuffer = async ({
circularBufferSize = DEFAULT_CBUF_SIZE,
stateDir = '/tmp',
circularBufferFilename,
}) => {
const filename = circularBufferFilename || `${stateDir}/${DEFAULT_CBUF_FILE}`;

const newArenaSize = await initializeCircularBuffer(
filename,
circularBufferSize,
);

/**
* @type {Uint8Array}
* BufferFromFile mmap()s the file into the process address space.
*/
const fileBuf = BufferFromFile(filename).Uint8Array();
const header = new DataView(fileBuf.buffer, 0, I_ARENA_START);

// Detect the arena size from the header, if not initialized.
const hdrArenaSize = header.getBigUint64(I_ARENA_SIZE);
const arenaSize = newArenaSize || hdrArenaSize;

const hdrMagic = header.getBigUint64(I_MAGIC);
SLOG_MAGIC === hdrMagic ||
Fail`${filename} is not a slog buffer; wanted magic ${SLOG_MAGIC}, got ${hdrMagic}`;
arenaSize === hdrArenaSize ||
Fail`${filename} arena size mismatch; wanted ${arenaSize}, got ${hdrArenaSize}`;
const arena = new Uint8Array(
fileBuf.buffer,
header.byteLength,
Number(arenaSize),
);

/**
*
* @param {Uint8Array} outbuf
* @param {number} readStart
* @param {number} firstReadLength
*/

function readRecord(outbuf, readStart, firstReadLength) {
outbuf.set(arena.subarray(readStart, readStart + firstReadLength));
if (firstReadLength < outbuf.byteLength) {
outbuf.set(
arena.subarray(0, outbuf.byteLength - firstReadLength),
firstReadLength,
);
}
}

/**
*
* @param {Uint8Array} record
* @param {number} firstWriteLength
* @param {bigint} circEnd
*/
const writeRecord = (record, firstWriteLength, circEnd) => {
arena.set(record.subarray(0, firstWriteLength), Number(circEnd));
if (firstWriteLength < record.byteLength) {
// Write to the beginning of the arena.
arena.set(record.subarray(firstWriteLength, record.byteLength), 0);
}
return Promise.resolve();
};

/**
* @param {Uint8Array} outbuf
* @param {number} [offset] offset relative to the current trailing edge (circStart) of the data
* @returns {IteratorResult<Uint8Array, void>}
*/
return finishCircularBuffer(arenaSize, header, readRecord, writeRecord);
};

/**
*
* @param {Pick<Awaited<ReturnType<typeof makeSimpleCircularBuffer>>, 'writeCircBuf'>} circBuf
Expand Down
4 changes: 2 additions & 2 deletions packages/telemetry/src/frcat-entrypoint.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import '@endo/init';

import { makeMemoryMappedCircularBuffer } from './flight-recorder.js';
import { makeSimpleCircularBuffer } from './flight-recorder.js';

const main = async () => {
const files = process.argv.slice(2);
Expand All @@ -14,7 +14,7 @@ const main = async () => {
}

for await (const file of files) {
const { readCircBuf } = await makeMemoryMappedCircularBuffer({
const { readCircBuf } = await makeSimpleCircularBuffer({
circularBufferFilename: file,
circularBufferSize: 0,
});
Expand Down
5 changes: 1 addition & 4 deletions packages/telemetry/test/test-flight-recorder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ import tmp from 'tmp';
import { test } from './prepare-test-env-ava.js';

import {
makeMemoryMappedCircularBuffer,
makeSimpleCircularBuffer,
makeSlogSenderFromBuffer,
} from '../src/flight-recorder.js';

// Factored this way to support multiple implementations, which at one point there were
const bufferTests = test.macro(
/**
*
Expand Down Expand Up @@ -78,9 +78,6 @@ const bufferTests = test.macro(
},
);

test('memory mapped', bufferTests, {
makeBuffer: makeMemoryMappedCircularBuffer,
});
test('simple', bufferTests, {
makeBuffer: makeSimpleCircularBuffer,
});
9 changes: 0 additions & 9 deletions patches/bufferfromfile+0.4.377.patch
View file

This file was deleted.

78 changes: 6 additions & 72 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1624,21 +1624,6 @@
npmlog "^6.0.2"
write-file-atomic "^4.0.1"

"@mapbox/node-pre-gyp@1.0.9":
version "1.0.9"
resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.9.tgz#09a8781a3a036151cdebbe8719d6f8b25d4058bc"
integrity sha512-aDF3S3rK9Q2gey/WAttUlISduDItz5BU3306M9Eyv6/oS40aMprnopshtlKTykxRNIBEZuRMaZAnbrQ4QtKGyw==
dependencies:
detect-libc "^2.0.0"
https-proxy-agent "^5.0.0"
make-dir "^3.1.0"
node-fetch "^2.6.7"
nopt "^5.0.0"
npmlog "^5.0.1"
rimraf "^3.0.2"
semver "^7.3.5"
tar "^6.1.11"

"@noble/hashes@^1", "@noble/hashes@^1.0.0":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.1.2.tgz#e9e035b9b166ca0af657a7848eb2718f0f22f183"
Expand Down Expand Up @@ -2883,14 +2868,6 @@ are-docs-informative@^0.0.2:
resolved "https://registry.yarnpkg.com/are-docs-informative/-/are-docs-informative-0.0.2.tgz#387f0e93f5d45280373d387a59d34c96db321963"
integrity sha512-ixiS0nLNNG5jNQzgZJNoUpBKdo9yTYZMGJ+QgT2jmjR7G7+QHRCc4v6LQ3NgE7EBJq+o0ams3waJwkrlBom8Ig==

are-we-there-yet@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz#372e0e7bd279d8e94c653aaa1f67200884bf3e1c"
integrity sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==
dependencies:
delegates "^1.0.0"
readable-stream "^3.6.0"

are-we-there-yet@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz#679df222b278c64f2cdba1175cdc00b0d96164bd"
Expand Down Expand Up @@ -3299,14 +3276,6 @@ buffer@^5.5.0:
base64-js "^1.3.1"
ieee754 "^1.1.13"

bufferfromfile@agoric-labs/BufferFromFile#Agoric-built:
version "0.4.377"
resolved "https://codeload.github.com/agoric-labs/BufferFromFile/tar.gz/691031035856fadba2603e52f4411160d7f34ed6"
dependencies:
"@mapbox/node-pre-gyp" "1.0.9"
node-gyp "^9.3.1"
wbasenodejscpp latest

builtin-modules@^3.1.0, builtin-modules@^3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.3.0.tgz#cae62812b89801e9656336e46223e030386be7b6"
Expand Down Expand Up @@ -3647,7 +3616,7 @@ color-string@^1.6.0:
color-name "^1.0.0"
simple-swizzle "^0.2.2"

color-support@^1.1.2, color-support@^1.1.3:
color-support@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2"
integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==
Expand Down Expand Up @@ -3768,7 +3737,7 @@ confusing-browser-globals@^1.0.10:
resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81"
integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==

console-control-strings@^1.0.0, console-control-strings@^1.1.0:
console-control-strings@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=
Expand Down Expand Up @@ -5253,21 +5222,6 @@ functions-have-names@^1.2.2:
resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.2.tgz#98d93991c39da9361f8e50b337c4f6e41f120e21"
integrity sha512-bLgc3asbWdwPbx2mNk2S49kmJCuQeu0nfmaOgbs8WIyzzkw3r4htszdIi9Q9EMezDPTYuJx2wvjZ/EwgAthpnA==

gauge@^3.0.0:
version "3.0.2"
resolved "https://registry.yarnpkg.com/gauge/-/gauge-3.0.2.tgz#03bf4441c044383908bcfa0656ad91803259b395"
integrity sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==
dependencies:
aproba "^1.0.3 || ^2.0.0"
color-support "^1.1.2"
console-control-strings "^1.0.0"
has-unicode "^2.0.1"
object-assign "^4.1.1"
signal-exit "^3.0.0"
string-width "^4.2.3"
strip-ansi "^6.0.1"
wide-align "^1.1.2"

gauge@^4.0.3:
version "4.0.4"
resolved "https://registry.yarnpkg.com/gauge/-/gauge-4.0.4.tgz#52ff0652f2bbf607a989793d53b751bef2328dce"
Expand Down Expand Up @@ -6681,7 +6635,7 @@ make-dir@^2.1.0:
pify "^4.0.1"
semver "^5.6.0"

make-dir@^3.0.0, make-dir@^3.1.0:
make-dir@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==
Expand Down Expand Up @@ -7422,7 +7376,7 @@ node-gyp-build@^4.3.0, node-gyp-build@^4.4.0, node-gyp-build@^4.5.0:
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.0.tgz#0c52e4cbf54bbd28b709820ef7b6a3c2d6209055"
integrity sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==

node-gyp@^9.0.0, node-gyp@^9.3.1:
node-gyp@^9.0.0:
version "9.4.0"
resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.0.tgz#2a7a91c7cba4eccfd95e949369f27c9ba704f369"
integrity sha512-dMXsYP6gc9rRbejLXmTbVRYjAHw7ppswsKyMxuxJxxOHzluIO1rGp9TOQgjFJ+2MCqcOcQTOPB/8Xwhr+7s4Eg==
Expand Down Expand Up @@ -7610,16 +7564,6 @@ npm-run-path@^5.1.0:
dependencies:
path-key "^4.0.0"

npmlog@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-5.0.1.tgz#f06678e80e29419ad67ab964e0fa69959c1eb8b0"
integrity sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==
dependencies:
are-we-there-yet "^2.0.0"
console-control-strings "^1.1.0"
gauge "^3.0.0"
set-blocking "^2.0.0"

npmlog@^6.0.0, npmlog@^6.0.2:
version "6.0.2"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-6.0.2.tgz#c8166017a42f2dea92d6453168dd865186a70830"
Expand Down Expand Up @@ -7681,11 +7625,6 @@ nx@15.9.4, "nx@>=14.8.1 < 16":
"@nrwl/nx-win32-arm64-msvc" "15.9.4"
"@nrwl/nx-win32-x64-msvc" "15.9.4"

object-assign@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=

object-hash@^1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df"
Expand Down Expand Up @@ -8937,7 +8876,7 @@ side-channel@^1.0.4:
get-intrinsic "^1.0.2"
object-inspect "^1.9.0"

signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7:
signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7:
version "3.0.7"
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==
Expand Down Expand Up @@ -9877,11 +9816,6 @@ walk-up-path@^1.0.0:
resolved "https://registry.yarnpkg.com/walk-up-path/-/walk-up-path-1.0.0.tgz#d4745e893dd5fd0dbb58dd0a4c6a33d9c9fec53e"
integrity sha512-hwj/qMDUEjCU5h0xr90KGCf0tg0/LgJbmOWgrWKYlcJZM7XvquvUJZ0G/HMGr7F7OQMOUuPHWP9JpriinkAlkg==

wbasenodejscpp@latest:
version "0.3.134"
resolved "https://registry.yarnpkg.com/wbasenodejscpp/-/wbasenodejscpp-0.3.134.tgz#c23c06bd8c3e170afe59084cfa6e7d41c07b0ffa"
integrity sha512-B3cyJ14XNKe7gfu+pQPdCkDKTMu4DiFS/iKWE+1AhroChj+hLiNYCcoUuvsjEocQtBPItjb57mbhdKCJ80pjCA==

wcwidth@^1.0.0, wcwidth@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8"
Expand Down Expand Up @@ -9944,7 +9878,7 @@ which@^2.0.1, which@^2.0.2:
dependencies:
isexe "^2.0.0"

wide-align@^1.1.2, wide-align@^1.1.5:
wide-align@^1.1.5:
version "1.1.5"
resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.5.tgz#df1d4c206854369ecf3c9a4898f1b23fbd9d15d3"
integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==
Expand Down

0 comments on commit 515ef12

Please sign in to comment.