Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: no-de-no-de, now with extra buns #9683

Merged
merged 10 commits into from
Jul 17, 2023
10 changes: 9 additions & 1 deletion packages/rest/.eslintrc.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
{
"extends": "../../.eslintrc.json"
"extends": "../../.eslintrc.json",
"rules": {
"n/prefer-global/url": "off",
"n/prefer-global/url-search-params": "off",
"n/prefer-global/buffer": "off",
"n/prefer-global/process": "off",
"no-restricted-globals": "off",
"unicorn/prefer-node-protocol": "off"
vladfrangu marked this conversation as resolved.
Show resolved Hide resolved
}
}
5 changes: 4 additions & 1 deletion packages/rest/.lintstagedrc.js
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
module.exports = require('../../.lintstagedrc.json');
module.exports = {
...require('../../.lintstagedrc.json'),
'src/**.ts': 'vitest related --run',
suneettipirneni marked this conversation as resolved.
Show resolved Hide resolved
};
4 changes: 4 additions & 0 deletions packages/rest/__tests__/setup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import { setDefaultStrategy } from '../src/environment.js';
import { makeRequest } from '../src/strategies/undiciRequest.js';

setDefaultStrategy(makeRequest);
20 changes: 13 additions & 7 deletions packages/rest/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,19 @@
"changelog": "git cliff --prepend ./CHANGELOG.md -u -c ./cliff.toml -r ../../ --include-path 'packages/rest/*'",
"release": "cliff-jumper"
},
"main": "./dist/index.js",
"module": "./dist/index.mjs",
"typings": "./dist/index.d.ts",
"types": "./dist/agnostic.d.ts",
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.mjs",
"require": "./dist/index.js"
"node": {
"types": "./dist/agnostic.d.ts",
"import": "./dist/agnostic.mjs",
"require": "./dist/agnostic.js"
},
"default": {
"types": "./dist/web.d.ts",
"import": "./dist/web.mjs",
"require": "./dist/web.js"
}
},
"./*": {
"types": "./dist/strategies/*.d.ts",
Expand Down Expand Up @@ -65,8 +70,9 @@
"@discordjs/util": "workspace:^",
"@sapphire/async-queue": "^1.5.0",
"@sapphire/snowflake": "^3.5.1",
"@vladfrangu/async_event_emitter": "^2.2.2",
"discord-api-types": "^0.37.45",
"file-type": "^18.4.0",
"magic-bytes.js": "^1.0.14",
"tslib": "^2.5.2",
"undici": "^5.22.1"
},
Expand Down
7 changes: 7 additions & 0 deletions packages/rest/src/agnostic.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { shouldUseGlobalFetchAndWebSocket } from '@discordjs/util';
import { setDefaultStrategy } from './environment.js';
import { makeRequest } from './strategies/undiciRequest.js';

setDefaultStrategy(shouldUseGlobalFetchAndWebSocket() ? fetch : makeRequest);

export * from './index.js';
11 changes: 11 additions & 0 deletions packages/rest/src/environment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import type { RESTOptions } from './index.js';

let defaultStrategy: RESTOptions['makeRequest'];

export function setDefaultStrategy(newStrategy: RESTOptions['makeRequest']) {
defaultStrategy = newStrategy;
}

export function getDefaultStrategy() {
return defaultStrategy;
}
2 changes: 0 additions & 2 deletions packages/rest/src/lib/CDN.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
/* eslint-disable jsdoc/check-param-names */

import { URL } from 'node:url';
import {
ALLOWED_EXTENSIONS,
ALLOWED_SIZES,
Expand Down
31 changes: 10 additions & 21 deletions packages/rest/src/lib/REST.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { EventEmitter } from 'node:events';
import type { Readable } from 'node:stream';
import type { ReadableStream } from 'node:stream/web';
import type { Collection } from '@discordjs/collection';
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
import type { Dispatcher, RequestInit, Response } from 'undici';
import { CDN } from './CDN.js';
import {
Expand Down Expand Up @@ -204,7 +204,7 @@ export interface APIRequest {
}

export interface ResponseLike
extends Pick<Response, 'arrayBuffer' | 'bodyUsed' | 'headers' | 'json' | 'ok' | 'status' | 'text'> {
extends Pick<Response, 'arrayBuffer' | 'bodyUsed' | 'headers' | 'json' | 'ok' | 'status' | 'statusText' | 'text'> {
body: Readable | ReadableStream | null;
}

Expand All @@ -223,31 +223,16 @@ export interface RestEvents {
handlerSweep: [sweptHandlers: Collection<string, IHandler>];
hashSweep: [sweptHashes: Collection<string, HashData>];
invalidRequestWarning: [invalidRequestInfo: InvalidRequestWarningData];
newListener: [name: string, listener: (...args: any) => void];
rateLimited: [rateLimitInfo: RateLimitData];
removeListener: [name: string, listener: (...args: any) => void];
response: [request: APIRequest, response: ResponseLike];
restDebug: [info: string];
}

export interface REST {
emit: (<K extends keyof RestEvents>(event: K, ...args: RestEvents[K]) => boolean) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, ...args: any[]) => boolean);
export type RestEventsMap = {
[K in keyof RestEvents]: RestEvents[K];
};

off: (<K extends keyof RestEvents>(event: K, listener: (...args: RestEvents[K]) => void) => this) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, listener: (...args: any[]) => void) => this);

on: (<K extends keyof RestEvents>(event: K, listener: (...args: RestEvents[K]) => void) => this) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, listener: (...args: any[]) => void) => this);

once: (<K extends keyof RestEvents>(event: K, listener: (...args: RestEvents[K]) => void) => this) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, listener: (...args: any[]) => void) => this);

removeAllListeners: (<K extends keyof RestEvents>(event?: K) => this) &
(<S extends string | symbol>(event?: Exclude<S, keyof RestEvents>) => this);
}

export class REST extends EventEmitter {
export class REST extends AsyncEventEmitter<RestEventsMap> {
public readonly cdn: CDN;

public readonly requestManager: RequestManager;
Expand All @@ -256,9 +241,13 @@ export class REST extends EventEmitter {
super();
this.cdn = new CDN(options.cdn ?? DefaultRestOptions.cdn);
this.requestManager = new RequestManager(options)
// @ts-expect-error For some reason ts can't infer these types
.on(RESTEvents.Debug, this.emit.bind(this, RESTEvents.Debug))
// @ts-expect-error For some reason ts can't infer these types
.on(RESTEvents.RateLimited, this.emit.bind(this, RESTEvents.RateLimited))
// @ts-expect-error For some reason ts can't infer these types
.on(RESTEvents.InvalidRequestWarning, this.emit.bind(this, RESTEvents.InvalidRequestWarning))
// @ts-expect-error For some reason ts can't infer these types
.on(RESTEvents.HashSweep, this.emit.bind(this, RESTEvents.HashSweep));

this.on('newListener', (name, listener) => {
Expand Down
58 changes: 22 additions & 36 deletions packages/rest/src/lib/RequestManager.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { Blob, Buffer } from 'node:buffer';
import { EventEmitter } from 'node:events';
import { setInterval, clearInterval } from 'node:timers';
import type { URLSearchParams } from 'node:url';
import { Collection } from '@discordjs/collection';
import { lazy } from '@discordjs/util';
import { DiscordSnowflake } from '@sapphire/snowflake';
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
import { filetypeinfo } from 'magic-bytes.js';
import type { RequestInit, BodyInit, Dispatcher, Agent } from 'undici';
import type { RESTOptions, ResponseLike, RestEvents } from './REST.js';
import type { RESTOptions, ResponseLike, RestEventsMap } from './REST.js';
import { BurstHandler } from './handlers/BurstHandler.js';
import { SequentialHandler } from './handlers/SequentialHandler.js';
import type { IHandler } from './interfaces/Handler.js';
Expand All @@ -17,9 +14,7 @@ import {
OverwrittenMimeTypes,
RESTEvents,
} from './utils/constants.js';

// Make this a lazy dynamic import as file-type is a pure ESM package
const getFileType = lazy(async () => import('file-type'));
import { isBufferLike } from './utils/utils.js';

/**
* Represents a file to be added to the request
Expand All @@ -32,7 +27,7 @@ export interface RawFile {
/**
* The actual data for the file
*/
data: Buffer | boolean | number | string;
data: Buffer | Uint8Array | boolean | number | string;
/**
* An explicit key to use for key of the formdata field for this file.
* When not provided, the index of the file in the files array is used in the form `files[${index}]`.
Expand Down Expand Up @@ -162,27 +157,10 @@ export interface HashData {
value: string;
}

export interface RequestManager {
emit: (<K extends keyof RestEvents>(event: K, ...args: RestEvents[K]) => boolean) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, ...args: any[]) => boolean);

off: (<K extends keyof RestEvents>(event: K, listener: (...args: RestEvents[K]) => void) => this) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, listener: (...args: any[]) => void) => this);

on: (<K extends keyof RestEvents>(event: K, listener: (...args: RestEvents[K]) => void) => this) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, listener: (...args: any[]) => void) => this);

once: (<K extends keyof RestEvents>(event: K, listener: (...args: RestEvents[K]) => void) => this) &
(<S extends string | symbol>(event: Exclude<S, keyof RestEvents>, listener: (...args: any[]) => void) => this);

removeAllListeners: (<K extends keyof RestEvents>(event?: K) => this) &
(<S extends string | symbol>(event?: Exclude<S, keyof RestEvents>) => this);
}

/**
* Represents the class that manages handlers for endpoints
*/
export class RequestManager extends EventEmitter {
export class RequestManager extends AsyncEventEmitter<RestEventsMap> {
/**
* The {@link https://undici.nodejs.org/#/docs/api/Agent | Agent} for all requests
* performed by this manager.
Expand Down Expand Up @@ -216,9 +194,9 @@ export class RequestManager extends EventEmitter {

#token: string | null = null;

private hashTimer!: NodeJS.Timer;
private hashTimer!: NodeJS.Timer | number;

private handlerTimer!: NodeJS.Timer;
private handlerTimer!: NodeJS.Timer | number;

public readonly options: RESTOptions;

Expand Down Expand Up @@ -269,7 +247,9 @@ export class RequestManager extends EventEmitter {

// Fire event
this.emit(RESTEvents.HashSweep, sweptHashes);
}, this.options.hashSweepInterval).unref();
}, this.options.hashSweepInterval);

this.hashTimer.unref?.();
}

if (this.options.handlerSweepInterval !== 0 && this.options.handlerSweepInterval !== Number.POSITIVE_INFINITY) {
Expand All @@ -292,7 +272,9 @@ export class RequestManager extends EventEmitter {

// Fire event
this.emit(RESTEvents.HandlerSweep, sweptHandlers);
}, this.options.handlerSweepInterval).unref();
}, this.options.handlerSweepInterval);

this.handlerTimer.unref?.();
}
}

Expand Down Expand Up @@ -425,14 +407,18 @@ export class RequestManager extends EventEmitter {
// FormData.append only accepts a string or Blob.
// https://developer.mozilla.org/en-US/docs/Web/API/Blob/Blob#parameters
// The Blob constructor accepts TypedArray/ArrayBuffer, strings, and Blobs.
if (Buffer.isBuffer(file.data)) {
if (isBufferLike(file.data)) {
// Try to infer the content type from the buffer if one isn't passed
const { fileTypeFromBuffer } = await getFileType();
let contentType = file.contentType;

if (!contentType) {
const parsedType = (await fileTypeFromBuffer(file.data))?.mime;
const [parsedType] = filetypeinfo(file.data);

if (parsedType) {
contentType = OverwrittenMimeTypes[parsedType as keyof typeof OverwrittenMimeTypes] ?? parsedType;
contentType =
OverwrittenMimeTypes[parsedType.mime as keyof typeof OverwrittenMimeTypes] ??
parsedType.mime ??
'application/octet-stream';
}
}

Expand Down
6 changes: 3 additions & 3 deletions packages/rest/src/lib/errors/HTTPError.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { STATUS_CODES } from 'node:http';
import type { InternalRequest } from '../RequestManager.js';
import type { RequestBody } from './DiscordAPIError.js';

Expand All @@ -12,18 +11,19 @@ export class HTTPError extends Error {

/**
* @param status - The status code of the response
* @param statusText - The status text of the response
* @param method - The method of the request that erred
* @param url - The url of the request that erred
* @param bodyData - The unparsed data for the request that errored
*/
public constructor(
public status: number,
statusText: string,
public method: string,
public url: string,
bodyData: Pick<InternalRequest, 'body' | 'files'>,
) {
super(STATUS_CODES[status]);

super(statusText);
this.requestBody = { files: bodyData.files, json: bodyData.body };
}
}
3 changes: 1 addition & 2 deletions packages/rest/src/lib/handlers/BurstHandler.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
import { setTimeout as sleep } from 'node:timers/promises';
import type { RequestInit } from 'undici';
import type { ResponseLike } from '../REST.js';
import type { HandlerRequestData, RequestManager, RouteData } from '../RequestManager.js';
import type { IHandler } from '../interfaces/Handler.js';
import { RESTEvents } from '../utils/constants.js';
import { onRateLimit } from '../utils/utils.js';
import { onRateLimit, sleep } from '../utils/utils.js';
import { handleErrors, incrementInvalidCount, makeNetworkRequest } from './Shared.js';

/**
Expand Down
3 changes: 1 addition & 2 deletions packages/rest/src/lib/handlers/SequentialHandler.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import { setTimeout as sleep } from 'node:timers/promises';
import { AsyncQueue } from '@sapphire/async-queue';
import type { RequestInit } from 'undici';
import type { RateLimitData, ResponseLike } from '../REST.js';
import type { HandlerRequestData, RequestManager, RouteData } from '../RequestManager.js';
import type { IHandler } from '../interfaces/Handler.js';
import { RESTEvents } from '../utils/constants.js';
import { hasSublimit, onRateLimit } from '../utils/utils.js';
import { hasSublimit, onRateLimit, sleep } from '../utils/utils.js';
import { handleErrors, incrementInvalidCount, makeNetworkRequest } from './Shared.js';

const enum QueueType {
Expand Down
6 changes: 2 additions & 4 deletions packages/rest/src/lib/handlers/Shared.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import { setTimeout, clearTimeout } from 'node:timers';
import { Response } from 'undici';
import type { RequestInit } from 'undici';
import type { ResponseLike } from '../REST.js';
import type { HandlerRequestData, RequestManager, RouteData } from '../RequestManager.js';
Expand Down Expand Up @@ -65,7 +63,7 @@ export async function makeNetworkRequest(
retries: number,
) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), manager.options.timeout).unref();
const timeout = setTimeout(() => controller.abort(), manager.options.timeout);
vladfrangu marked this conversation as resolved.
Show resolved Hide resolved
if (requestData.signal) {
// If the user signal was aborted, abort the controller, else abort the local signal.
// The reason why we don't re-use the user's signal, is because users may use the same signal for multiple
Expand Down Expand Up @@ -135,7 +133,7 @@ export async function handleErrors(
}

// We are out of retries, throw an error
throw new HTTPError(status, method, url, requestData);
throw new HTTPError(status, res.statusText, method, url, requestData);
} else {
// Handle possible malformed requests
if (status >= 400 && status < 500) {
Expand Down
17 changes: 6 additions & 11 deletions packages/rest/src/lib/utils/constants.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
import process from 'node:process';
import { lazy } from '@discordjs/util';
import { getUserAgentAppendix } from '@discordjs/util';
import { APIVersion } from 'discord-api-types/v10';
import type { RESTOptions } from '../REST.js';

const getUndiciRequest = lazy(async () => {
return import('../../strategies/undiciRequest.js');
});
import { getDefaultStrategy } from '../../environment.js';
import type { RESTOptions, ResponseLike } from '../REST.js';

export const DefaultUserAgent =
`DiscordBot (https://discord.js.org, [VI]{{inject}}[/VI])` as `DiscordBot (https://discord.js.org, ${string})`;

/**
* The default string to append onto the user agent.
*/
export const DefaultUserAgentAppendix = process.release?.name === 'node' ? `Node.js/${process.version}` : '';
export const DefaultUserAgentAppendix = getUserAgentAppendix();

export const DefaultRestOptions = {
agent: null,
Expand All @@ -32,9 +28,8 @@ export const DefaultRestOptions = {
hashSweepInterval: 14_400_000, // 4 Hours
hashLifetime: 86_400_000, // 24 Hours
handlerSweepInterval: 3_600_000, // 1 Hour
async makeRequest(...args) {
const strategy = await getUndiciRequest();
return strategy.makeRequest(...args);
async makeRequest(...args): Promise<ResponseLike> {
return getDefaultStrategy()(...args);
},
} as const satisfies Required<RESTOptions>;

Expand Down
Loading