Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

reduce encoding serialization cost #2478

Merged
merged 1 commit into from
Aug 6, 2024
Merged

reduce encoding serialization cost #2478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions src/node/internal/buffer.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,42 @@ interface CompareOptions {

type BufferSource = ArrayBufferView | ArrayBuffer;

export type Encoding = number;

export function byteLength(value: string): number;
export function compare(a: Uint8Array, b: Uint8Array, options?: CompareOptions): number;
export function concat(list: Uint8Array[], length: number): ArrayBuffer;
export function decodeString(value: string, encoding: string): ArrayBuffer;
export function decodeString(value: string, encoding: Encoding): ArrayBuffer;
export function fillImpl(buffer: Uint8Array,
value: string | BufferSource,
start: number,
end: number,
encoding?: string): void;
encoding?: Encoding): void;
export function indexOf(buffer: Uint8Array,
value: string | Uint8Array,
byteOffset?: number,
encoding?: string,
encoding?: Encoding,
findLast?: boolean): number | undefined;
export function swap(buffer: Uint8Array, size: 16|32|64): void;
export function toString(buffer: Uint8Array,
start: number,
end: number,
encoding: string): string;
encoding: Encoding): string;
export function write(buffer: Uint8Array,
value: string,
offset: number,
length: number,
encoding: string): void;
encoding: Encoding): void;
export function decode(buffer: Uint8Array, state: Uint8Array): string;
export function flush(state: Uint8Array): string;
export function isAscii(value: ArrayBufferView): boolean;
export function isUtf8(value: ArrayBufferView): boolean;
export function transcode(source: ArrayBufferView, fromEncoding: string, toEncoding: string): ArrayBuffer;
export function transcode(source: ArrayBufferView, fromEncoding: Encoding, toEncoding: Encoding): ArrayBuffer;

export const ASCII: Encoding;
export const LATIN1: Encoding;
export const UTF8: Encoding;
export const UTF16LE: Encoding;
export const BASE64: Encoding;
export const BASE64URL: Encoding;
export const HEX: Encoding;
10 changes: 0 additions & 10 deletions src/node/internal/crypto_hash.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,10 @@ import {
} from 'node-internal:internal_errors';

import {
validateEncoding,
validateString,
validateUint32,
} from 'node-internal:validators';

import {
normalizeEncoding
} from 'node-internal:internal_utils';

import {
isArrayBufferView,
isCryptoKey,
Expand Down Expand Up @@ -129,9 +124,6 @@ Hash.prototype.copy = function(this: Hash, options?: HashOptions): Hash {
Hash.prototype._transform = function(this: Hash | Hmac, chunk: string | Buffer | ArrayBufferView,
encoding: string, callback: TransformCallback): void {
if (typeof chunk === 'string') {
encoding ??= 'utf-8';
validateEncoding(chunk, encoding);
encoding = normalizeEncoding(encoding)!;
chunk = Buffer.from(chunk, encoding);
}
this[kHandle].update(chunk);
Expand All @@ -155,8 +147,6 @@ Hash.prototype.update = function(this: Hash | Hmac, data: string | Buffer | Arra
throw new ERR_CRYPTO_HASH_FINALIZED();

if (typeof data === 'string') {
validateEncoding(data, encoding!);
encoding = normalizeEncoding(encoding);
data = Buffer.from(data, encoding);
} else if (!isArrayBufferView(data)) {
throw new ERR_INVALID_ARG_TYPE(
Expand Down
91 changes: 45 additions & 46 deletions src/node/internal/internal_buffer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
} from 'node-internal:internal_errors';

import { default as bufferUtil } from 'node-internal:buffer';
import type { Encoding } from 'node-internal:buffer';

import {
isAnyArrayBuffer,
Expand All @@ -41,6 +42,8 @@ import {
inspect as utilInspect,
} from 'node-internal:internal_inspect';

const { ASCII, BASE64, BASE64URL, HEX, LATIN1, UTF16LE, UTF8 } = bufferUtil;

// Temporary buffers to convert numbers.
const float32Array = new Float32Array(1);
const uInt8Float32Array = new Uint8Array(float32Array.buffer);
Expand Down Expand Up @@ -313,11 +316,11 @@ function fromString(string: StringLike, encoding?: string) {
encoding = "utf8";
}
const normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

const ab = bufferUtil.decodeString(`${string}`, normalizedEncoding as string);
const ab = bufferUtil.decodeString(`${string}`, normalizedEncoding);
if (ab === undefined) {
throw new ERR_INVALID_ARG_VALUE('string', string,
`Unable to decode string using encoding ${encoding}`);
Expand Down Expand Up @@ -493,25 +496,22 @@ function byteLength(string: string|ArrayBufferView|ArrayBuffer|SharedArrayBuffer
}

string = `${string}`;
let normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
normalizedEncoding = "utf8";
}
const normalizedEncoding = normalizeEncoding(encoding) ?? UTF8;

switch (normalizedEncoding) {
case 'ascii':
case ASCII:
// Fall through
case 'latin1':
case LATIN1:
return (string as string).length;
case 'utf16le':
case UTF16LE:
return (string as string).length * 2;
case 'base64':
case BASE64:
// Fall through
case 'base64url':
case BASE64URL:
return base64ByteLength(string as string);
case 'hex':
case HEX:
return (string as string).length >>> 1;
case 'utf8':
case UTF8:
// Fall-through
default:
return bufferUtil.byteLength(string as string);
Expand Down Expand Up @@ -552,7 +552,7 @@ Buffer.prototype.toString = function toString(
start?: number,
end?: number) {
if (arguments.length === 0) {
return bufferUtil.toString(this, 0, this.length, "utf8");
return bufferUtil.toString(this, 0, this.length, UTF8);
}

const len = this.length;
Expand All @@ -576,11 +576,11 @@ Buffer.prototype.toString = function toString(
}

const normalizedEncoding = normalizeEncoding(`${encoding}`);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

return bufferUtil.toString(this, start as number, end as number, normalizedEncoding as string);
return bufferUtil.toString(this, start as number, end as number, normalizedEncoding);
};

Buffer.prototype.toLocaleString = Buffer.prototype.toString;
Expand Down Expand Up @@ -732,9 +732,9 @@ function bidirectionalIndexOf(
throw new ERR_INVALID_ARG_TYPE('value', ['number', 'string', 'Buffer', 'Uint8Array'], val);
}

let normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
const normalizedEncoding = normalizeEncoding(encoding);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

const result = bufferUtil.indexOf(buffer, val, byteOffset, normalizedEncoding, dir);
Expand All @@ -758,44 +758,44 @@ Buffer.prototype.lastIndexOf = function lastIndexOf(
Buffer.prototype.asciiSlice = function asciiSlice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'ascii');
return bufferUtil.toString(this, offset, offset + length, ASCII);
};

Buffer.prototype.base64Slice = function base64Slice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'base64');
return bufferUtil.toString(this, offset, offset + length, BASE64);
};

Buffer.prototype.base64urlSlice = function base64urlSlice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'base64url');
return bufferUtil.toString(this, offset, offset + length, BASE64URL);
};

Buffer.prototype.hexSlice = function hexSlice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'hex');
return bufferUtil.toString(this, offset, offset + length, HEX);
};

Buffer.prototype.latin1Slice = function latin1Slice(offset: number,
length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'latin1');
return bufferUtil.toString(this, offset, offset + length, LATIN1);
};

Buffer.prototype.ucs2Slice = function ucs2Slice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'utf16le');
return bufferUtil.toString(this, offset, offset + length, UTF16LE);
};

Buffer.prototype.utf8Slice = function utf8Slice(offset: number, length: number) {
validateOffset(offset, "offset", 0, this.length);
validateOffset(length, "length", 0, this.length - offset);
return bufferUtil.toString(this, offset, offset + length, 'utf8');
return bufferUtil.toString(this, offset, offset + length, UTF8);
};

Buffer.prototype.asciiWrite = function asciiWrite(string: StringLike,
Expand All @@ -805,7 +805,7 @@ Buffer.prototype.asciiWrite = function asciiWrite(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'ascii');
return bufferUtil.write(this, `${string}`, offset as number, length as number, ASCII);
};

Buffer.prototype.base64Write = function base64Write(string: StringLike,
Expand All @@ -815,7 +815,7 @@ Buffer.prototype.base64Write = function base64Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'base64');
return bufferUtil.write(this, `${string}`, offset as number, length as number, BASE64);
};

Buffer.prototype.base64urlWrite = function base64urlWrite(string: StringLike,
Expand All @@ -825,7 +825,7 @@ Buffer.prototype.base64urlWrite = function base64urlWrite(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'base64url');
return bufferUtil.write(this, `${string}`, offset as number, length as number, BASE64URL);
};

Buffer.prototype.hexWrite = function hexWrite(string: StringLike,
Expand All @@ -835,7 +835,7 @@ Buffer.prototype.hexWrite = function hexWrite(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'hex');
return bufferUtil.write(this, `${string}`, offset as number, length as number, HEX);
};

Buffer.prototype.latin1Write = function latin1Write(string: StringLike,
Expand All @@ -845,7 +845,7 @@ Buffer.prototype.latin1Write = function latin1Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'latin1');
return bufferUtil.write(this, `${string}`, offset as number, length as number, LATIN1);
};

Buffer.prototype.ucs2Write = function ucs2Write(string: StringLike,
Expand All @@ -855,7 +855,7 @@ Buffer.prototype.ucs2Write = function ucs2Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'utf16le');
return bufferUtil.write(this, `${string}`, offset as number, length as number, UTF16LE);
};

Buffer.prototype.utf8Write = function utf8Write(string: StringLike,
Expand All @@ -865,7 +865,7 @@ Buffer.prototype.utf8Write = function utf8Write(string: StringLike,
length ??= this.length;
validateOffset(offset as number, "offset", 0, this.length);
validateOffset(length as number, "length", 0, this.length - offset);
return bufferUtil.write(this, `${string}`, offset as number, length as number, 'utf8');
return bufferUtil.write(this, `${string}`, offset as number, length as number, UTF8);
};

Buffer.prototype.write = function write(string: StringLike,
Expand All @@ -875,7 +875,7 @@ Buffer.prototype.write = function write(string: StringLike,
string = `${string}`;
if (offset === undefined) {
// Buffer#write(string)
return bufferUtil.write(this, string as string, 0, this.length, "utf8");
return bufferUtil.write(this, string as string, 0, this.length, UTF8);
}

if (length === undefined && typeof offset === 'string') {
Expand Down Expand Up @@ -903,16 +903,16 @@ Buffer.prototype.write = function write(string: StringLike,
}

if (!encoding) {
return bufferUtil.write(this, string as string, offset as number, length as number, "utf8");
return bufferUtil.write(this, string as string, offset as number, length as number, UTF8);
}

const normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}

return bufferUtil.write(this, string as string, offset as number, length as number,
normalizedEncoding as string);
normalizedEncoding);
};

Buffer.prototype.toJSON = function toJSON() {
Expand Down Expand Up @@ -1520,7 +1520,7 @@ Buffer.prototype.fill = function fill(
start?: number|string,
end?: number,
encoding?: string) {
let normalizedEncoding : string | undefined;
let normalizedEncoding: Encoding | undefined;
if (typeof val === "string") {
if (typeof start === "string") {
encoding = start;
Expand All @@ -1531,8 +1531,8 @@ Buffer.prototype.fill = function fill(
end = this.length;
}
normalizedEncoding = normalizeEncoding(encoding);
if (!Buffer.isEncoding(normalizedEncoding)) {
throw new ERR_UNKNOWN_ENCODING(encoding as string);
if (normalizedEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(`${encoding}`);
}
if (val.length === 1) {
const code = val.charCodeAt(0);
Expand Down Expand Up @@ -2299,14 +2299,13 @@ export function transcode(source: ArrayBufferView, fromEncoding: string, toEncod
throw new ERR_INVALID_ARG_TYPE('source', 'ArrayBufferView', typeof source);
}
const normalizedFromEncoding = normalizeEncoding(fromEncoding);
if (!Buffer.isEncoding(normalizedFromEncoding)) {
if (normalizedFromEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(fromEncoding);
}
const normalizedToEncoding = normalizeEncoding(toEncoding);
if (!Buffer.isEncoding(normalizedToEncoding)) {
if (normalizedToEncoding === undefined) {
throw new ERR_UNKNOWN_ENCODING(toEncoding);
}
// TODO(soon): Optimization opportunity: Pass int encoding values instead of strings.
return Buffer.from(bufferUtil.transcode(source, normalizedFromEncoding, normalizedToEncoding));
}

Expand Down
Loading
Loading