Skip to content

Commit

Permalink
reduce encoding serialization cost
Browse files Browse the repository at this point in the history
  • Loading branch information
@anonrig
anonrig committed Aug 5, 2024
1 parent 17b52c5 commit d79e38f
Show file tree
Hide file tree
Showing 8 changed files with 139 additions and 144 deletions.
22 changes: 16 additions & 6 deletions src/node/internal/buffer.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,42 @@ interface CompareOptions {

type BufferSource = ArrayBufferView | ArrayBuffer;

export type Encoding = number;

export function byteLength(value: string): number;
export function compare(a: Uint8Array, b: Uint8Array, options?: CompareOptions): number;
export function concat(list: Uint8Array[], length: number): ArrayBuffer;
export function decodeString(value: string, encoding: string): ArrayBuffer;
export function decodeString(value: string, encoding: Encoding): ArrayBuffer;
export function fillImpl(buffer: Uint8Array,
value: string | BufferSource,
start: number,
end: number,
encoding?: string): void;
encoding?: Encoding): void;
export function indexOf(buffer: Uint8Array,
value: string | Uint8Array,
byteOffset?: number,
encoding?: string,
encoding?: Encoding,
findLast?: boolean): number | undefined;
export function swap(buffer: Uint8Array, size: 16|32|64): void;
export function toString(buffer: Uint8Array,
start: number,
end: number,
encoding: string): string;
encoding: Encoding): string;
export function write(buffer: Uint8Array,
value: string,
offset: number,
length: number,
encoding: string): void;
encoding: Encoding): void;
export function decode(buffer: Uint8Array, state: Uint8Array): string;
export function flush(state: Uint8Array): string;
export function isAscii(value: ArrayBufferView): boolean;
export function isUtf8(value: ArrayBufferView): boolean;
export function transcode(source: ArrayBufferView, fromEncoding: string, toEncoding: string): ArrayBuffer;
export function transcode(source: ArrayBufferView, fromEncoding: Encoding, toEncoding: Encoding): ArrayBuffer;

export const ASCII: Encoding;
export const LATIN1: Encoding;
export const UTF8: Encoding;
export const UTF16LE: Encoding;
export const BASE64: Encoding;
export const BASE64URL: Encoding;
export const HEX: Encoding;
10 changes: 0 additions & 10 deletions src/node/internal/crypto_hash.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,10 @@ import {
} from 'node-internal:internal_errors';

import {
validateEncoding,
validateString,
validateUint32,
} from 'node-internal:validators';

import {
normalizeEncoding
} from 'node-internal:internal_utils';

import {
isArrayBufferView,
isCryptoKey,
Expand Down Expand Up @@ -129,9 +124,6 @@ Hash.prototype.copy = function(this: Hash, options?: HashOptions): Hash {
Hash.prototype._transform = function(this: Hash | Hmac, chunk: string | Buffer | ArrayBufferView,
encoding: string, callback: TransformCallback): void {
if (typeof chunk === 'string') {
encoding ??= 'utf-8';
validateEncoding(chunk, encoding);
encoding = normalizeEncoding(encoding)!;
chunk = Buffer.from(chunk, encoding);
}
this[kHandle].update(chunk);
Expand All @@ -155,8 +147,6 @@ Hash.prototype.update = function(this: Hash | Hmac, data: string | Buffer | Arra
throw new ERR_CRYPTO_HASH_FINALIZED();

if (typeof data === 'string') {
validateEncoding(data, encoding!);
encoding = normalizeEncoding(encoding);
data = Buffer.from(data, encoding);
} else if (!isArrayBufferView(data)) {
throw new ERR_INVALID_ARG_TYPE(
Expand Down
91 changes: 45 additions & 46 deletions src/node/internal/internal_buffer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
} from 'node-internal:internal_errors';

import { default as bufferUtil } from 'node-internal:buffer';
import type { Encoding } from 'node-internal:buffer';

import {
isAnyArrayBuffer,
Expand All @@ -41,6 +42,8 @@ import {
inspect as utilInspect,
} from 'node-internal:internal_inspect';

const { ASCII, BASE64, BASE64URL, HEX, LATIN1, UTF16LE, UTF8 } = bufferUtil;

// Temporary buffers to convert numbers.
const float32Array = new Float32Array(1);
const uInt8Float32Array = new Uint8Array(float32Array.buffer);
Expand Down Expand Up @@ -313,11 +316,11 @@ function fromString(string: StringLike, encoding?: string) {
encoding = "utf8";
}
const normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

const ab = bufferUtil.decodeString(`${string}`, normalizedEncoding as string);
const ab = bufferUtil.decodeString(`${string}`, normalizedEncoding);
if (ab === undefined) {
throw new ERR_INVALID_ARG_VALUE('string', string,
`Unable to decode string using encoding ${encoding}`);
Expand Down Expand Up @@ -493,25 +496,22 @@ function byteLength(string: string|ArrayBufferView|ArrayBuffer|SharedArrayBuffer
}

string = `${string}`;
let normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
normalizedEncoding = "utf8";
}
const normalizedEncoding = normalizeEncoding(encoding) ?? UTF8;

switch (normalizedEncoding) {
case 'ascii':
case ASCII:
// Fall through
case 'latin1':
case LATIN1:
return (string as string).length;
case 'utf16le':
case UTF16LE:
return (string as string).length * 2;
case 'base64':
case BASE64:
// Fall through
case 'base64url':
case BASE64URL:
return base64ByteLength(string as string);
case 'hex':
case HEX:
return (string as string).length >>> 1;
case 'utf8':
case UTF8:
// Fall-through
default:
return bufferUtil.byteLength(string as string);
Expand Down Expand Up @@ -552,7 +552,7 @@ Buffer.prototype.toString = function toString(
start?: number,
end?: number) {
if (arguments.length === 0) {
return bufferUtil.toString(this, 0, this.length, "utf8");
return bufferUtil.toString(this, 0, this.length, UTF8);
}

const len = this.length;
Expand All @@ -576,11 +576,11 @@ Buffer.prototype.toString = function toString(
}

const normalizedEncoding = normalizeEncoding(`${encoding}`);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

return bufferUtil.toString(this, start as number, end as number, normalizedEncoding as string);
return bufferUtil.toString(this, start as number, end as number, normalizedEncoding);
};

Buffer.prototype.toLocaleString = Buffer.prototype.toString;
Expand Down Expand Up @@ -732,9 +732,9 @@ function bidirectionalIndexOf(
throw new ERR_INVALID_ARG_TYPE('value', ['number', 'string', 'Buffer', 'Uint8Array'], val);
}

let normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
const normalizedEncoding = normalizeEncoding(encoding);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

const result = bufferUtil.indexOf(buffer, val, byteOffset, normalizedEncoding, dir);
Expand All @@ -758,44 +758,44 @@ Buffer.prototype.lastIndexOf = function lastIndexOf(
Buffer.prototype.asciiSlice = function asciiSlice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'ascii');
return bufferUtil.toString(this, offset, offset + length, ASCII);
};

Buffer.prototype.base64Slice = function base64Slice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'base64');
return bufferUtil.toString(this, offset, offset + length, BASE64);
};

Buffer.prototype.base64urlSlice = function base64urlSlice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'base64url');
return bufferUtil.toString(this, offset, offset + length, BASE64URL);
};

Buffer.prototype.hexSlice = function hexSlice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'hex');
return bufferUtil.toString(this, offset, offset + length, HEX);
};

Buffer.prototype.latin1Slice = function latin1Slice(offset: number,
length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'latin1');
return bufferUtil.toString(this, offset, offset + length, LATIN1);
};

Buffer.prototype.ucs2Slice = function ucs2Slice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'utf16le');
return bufferUtil.toString(this, offset, offset + length, UTF16LE);
};

Buffer.prototype.utf8Slice = function utf8Slice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'utf8');
return bufferUtil.toString(this, offset, offset + length, UTF8);
};

Buffer.prototype.asciiWrite = function asciiWrite(string: StringLike,
Expand All @@ -805,7 +805,7 @@ Buffer.prototype.asciiWrite = function asciiWrite(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'ascii');
return bufferUtil.write(this, `${string}`, offset as number, length as number, ASCII);
};

Buffer.prototype.base64Write = function base64Write(string: StringLike,
Expand All @@ -815,7 +815,7 @@ Buffer.prototype.base64Write = function base64Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'base64');
return bufferUtil.write(this, `${string}`, offset as number, length as number, BASE64);
};

Buffer.prototype.base64urlWrite = function base64urlWrite(string: StringLike,
Expand All @@ -825,7 +825,7 @@ Buffer.prototype.base64urlWrite = function base64urlWrite(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'base64url');
return bufferUtil.write(this, `${string}`, offset as number, length as number, BASE64URL);
};

Buffer.prototype.hexWrite = function hexWrite(string: StringLike,
Expand All @@ -835,7 +835,7 @@ Buffer.prototype.hexWrite = function hexWrite(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'hex');
return bufferUtil.write(this, `${string}`, offset as number, length as number, HEX);
};

Buffer.prototype.latin1Write = function latin1Write(string: StringLike,
Expand All @@ -845,7 +845,7 @@ Buffer.prototype.latin1Write = function latin1Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'latin1');
return bufferUtil.write(this, `${string}`, offset as number, length as number, LATIN1);
};

Buffer.prototype.ucs2Write = function ucs2Write(string: StringLike,
Expand All @@ -855,7 +855,7 @@ Buffer.prototype.ucs2Write = function ucs2Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'utf16le');
return bufferUtil.write(this, `${string}`, offset as number, length as number, UTF16LE);
};

Buffer.prototype.utf8Write = function utf8Write(string: StringLike,
Expand All @@ -865,7 +865,7 @@ Buffer.prototype.utf8Write = function utf8Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'utf8');
return bufferUtil.write(this, `${string}`, offset as number, length as number, UTF8);
};

Buffer.prototype.write = function write(string: StringLike,
Expand All @@ -875,7 +875,7 @@ Buffer.prototype.write = function write(string: StringLike,
string = `${string}`;
if (offset === undefined) {
// Buffer#write(string)
return bufferUtil.write(this, string as string, 0, this.length, "utf8");
return bufferUtil.write(this, string as string, 0, this.length, UTF8);
}

if (length === undefined && typeof offset === 'string') {
Expand Down Expand Up @@ -903,16 +903,16 @@ Buffer.prototype.write = function write(string: StringLike,
}

if (!encoding) {
return bufferUtil.write(this, string as string, offset as number, length as number, "utf8");
return bufferUtil.write(this, string as string, offset as number, length as number, UTF8);
}

const normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

return bufferUtil.write(this, string as string, offset as number, length as number,
normalizedEncoding as string);
normalizedEncoding);
};

Buffer.prototype.toJSON = function toJSON() {
Expand Down Expand Up @@ -1520,7 +1520,7 @@ Buffer.prototype.fill = function fill(
start?: number|string,
end?: number,
encoding?: string) {
let normalizedEncoding : string | undefined;
let normalizedEncoding: Encoding | undefined;
if (typeof val === "string") {
if (typeof start === "string") {
encoding = start;
Expand All @@ -1531,8 +1531,8 @@ Buffer.prototype.fill = function fill(
end = this.length;
}
normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}
if (val.length === 1) {
const code = val.charCodeAt(0);
Expand Down Expand Up @@ -2299,14 +2299,13 @@ export function transcode(source: ArrayBufferView, fromEncoding: string, toEncod
throw new ERR_INVALID_ARG_TYPE('source', 'ArrayBufferView', typeof source);
}
const normalizedFromEncoding = normalizeEncoding(fromEncoding);
if (!Buffer.isEncoding(normalizedFromEncoding)) {
if (normalizedFromEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(fromEncoding);
}
const normalizedToEncoding = normalizeEncoding(toEncoding);
if (!Buffer.isEncoding(normalizedToEncoding)) {
if (normalizedToEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(toEncoding);
}
// TODO(soon): Optimization opportunity: Pass int encoding values instead of strings.
return Buffer.from(bufferUtil.transcode(source, normalizedFromEncoding, normalizedToEncoding));
}

Expand Down
Loading

0 comments on commit d79e38f

Please sign in to comment.