From f415d2f172110b0f0ad2fbab13c7b73d198c983c Mon Sep 17 00:00:00 2001 From: q_h Date: Thu, 11 Nov 2021 08:24:06 +0300 Subject: [PATCH] perf(dynamic-chunk): increase chunks time limits (#342) Changed the chunk time range from 2 - 8 sec to 8 - 24 sec. This noticeably reduces server and client overhead and decreases upload time. --- src/app/app.component.ts | 2 +- src/uploadx/lib/dynamic-chunk.spec.ts | 16 +++++++ src/uploadx/lib/dynamic-chunk.ts | 25 +++++++++++ src/uploadx/lib/uploader.ts | 62 ++++++++++++++------------- src/uploadx/lib/utils.spec.ts | 17 +------- src/uploadx/lib/utils.ts | 25 ----------- 6 files changed, 75 insertions(+), 72 deletions(-) create mode 100644 src/uploadx/lib/dynamic-chunk.spec.ts create mode 100644 src/uploadx/lib/dynamic-chunk.ts diff --git a/src/app/app.component.ts b/src/app/app.component.ts index 0a19f5fa..4edd47d7 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -20,6 +20,6 @@ export class AppComponent implements DoCheck { } ngDoCheck(): void { - console.log('change-detection ', this.changes++); + console.debug('change-detection count: ', this.changes++); } } diff --git a/src/uploadx/lib/dynamic-chunk.spec.ts b/src/uploadx/lib/dynamic-chunk.spec.ts new file mode 100644 index 00000000..fa54669f --- /dev/null +++ b/src/uploadx/lib/dynamic-chunk.spec.ts @@ -0,0 +1,16 @@ +import { DynamicChunk } from './dynamic-chunk'; + +describe('DynamicChunk', () => { + const init = DynamicChunk.size; + it('scale', () => { + expect(DynamicChunk.scale(0)).toEqual(init / 2); + expect(DynamicChunk.scale(0)).toEqual(init / 4); + expect(DynamicChunk.scale(Number.MAX_SAFE_INTEGER)).toEqual(init / 2); + expect(DynamicChunk.scale(Number.MAX_SAFE_INTEGER)).toEqual(init); + expect(DynamicChunk.scale(Number.MAX_SAFE_INTEGER)).toEqual(init * 2); + expect(DynamicChunk.scale(undefined as any)).toEqual(init * 2); + }); + afterEach(() => { + DynamicChunk.size = init; + }); +}); diff --git a/src/uploadx/lib/dynamic-chunk.ts b/src/uploadx/lib/dynamic-chunk.ts new file mode 100644 index 00000000..113baade --- /dev/null +++ b/src/uploadx/lib/dynamic-chunk.ts @@ -0,0 +1,25 @@ +const KiB = 1024; +/** + * Adaptive chunk size + */ +export class DynamicChunk { + /** Maximum chunk size in bytes */ + static maxSize = Number.MAX_SAFE_INTEGER; + /** Minimum chunk size in bytes */ + static minSize = 256 * KiB; + /** Initial chunk size in bytes */ + static size = 4 * (256 * KiB); + static minChunkTime = 8; + static maxChunkTime = 24; + + static scale(throughput: number): number { + const elapsedTime = DynamicChunk.size / throughput; + if (elapsedTime < DynamicChunk.minChunkTime) { + DynamicChunk.size = Math.min(DynamicChunk.maxSize, DynamicChunk.size * 2); + } + if (elapsedTime > DynamicChunk.maxChunkTime) { + DynamicChunk.size = Math.max(DynamicChunk.minSize, DynamicChunk.size / 2); + } + return DynamicChunk.size; + } +} diff --git a/src/uploadx/lib/uploader.ts b/src/uploadx/lib/uploader.ts index 35bcf288..371caef9 100644 --- a/src/uploadx/lib/uploader.ts +++ b/src/uploadx/lib/uploader.ts @@ -1,5 +1,6 @@ import { Ajax, AjaxRequestConfig } from './ajax'; import { Canceler } from './canceler'; +import { DynamicChunk } from './dynamic-chunk'; import { AuthorizeRequest, Metadata, @@ -16,7 +17,7 @@ import { } from './interfaces'; import { ErrorType, RetryHandler } from './retry-handler'; import { store } from './store'; -import { DynamicChunk, isNumber, unfunc } from './utils'; +import { isNumber, unfunc } from './utils'; const actionToStatusMap: { [K in UploadAction]: UploadStatus } = { pause: 'paused', @@ -58,6 +59,29 @@ export abstract class Uploader implements UploadState { private readonly _authorize: AuthorizeRequest; private readonly _prerequest: PreRequest; private _token!: string; + + constructor( + readonly file: File, + readonly options: Readonly, + readonly stateChange: (evt: UploadState) => void, + readonly ajax: Ajax + ) { + this.retry = new RetryHandler(options.retryConfig); + this.name = file.name; + this.size = file.size; + this.metadata = { + name: file.name, + mimeType: file.type || 'application/octet-stream', + size: file.size, + lastModified: + file.lastModified || (file as File & { lastModifiedDate: Date }).lastModifiedDate.getTime() + }; + options.maxChunkSize && (DynamicChunk.maxSize = options.maxChunkSize); + this._prerequest = options.prerequest || (req => req); + this._authorize = options.authorize || (req => req); + this.configure(options); + } + private _url = ''; get url(): string { @@ -88,28 +112,6 @@ export abstract class Uploader implements UploadState { } } - constructor( - readonly file: File, - readonly options: Readonly, - readonly stateChange: (evt: UploadState) => void, - readonly ajax: Ajax - ) { - this.retry = new RetryHandler(options.retryConfig); - this.name = file.name; - this.size = file.size; - this.metadata = { - name: file.name, - mimeType: file.type || 'application/octet-stream', - size: file.size, - lastModified: - file.lastModified || (file as File & { lastModifiedDate: Date }).lastModifiedDate.getTime() - }; - options.maxChunkSize && (DynamicChunk.maxSize = options.maxChunkSize); - this._prerequest = options.prerequest || (req => req); - this._authorize = options.authorize || (req => req); - this.configure(options); - } - /** * Configure uploader */ @@ -170,10 +172,6 @@ export abstract class Uploader implements UploadState { } } - private getRetryAfterFromBackend(): number { - return Number(this.getValueFromResponse('retry-after')) * 1000; - } - /** * Performs http requests */ @@ -264,20 +262,24 @@ export abstract class Uploader implements UploadState { return { start, end, body }; } + private getRetryAfterFromBackend(): number { + return Number(this.getValueFromResponse('retry-after')) * 1000; + } + private cleanup = () => store.delete(this.uploadId); private onProgress(): (evt: ProgressEvent) => void { let throttle: ReturnType | undefined; - const startTime = Date.now(); + const startTime = new Date().getTime(); return ({ loaded }) => { - const elapsedTime = (Date.now() - startTime) / 1000; + const elapsedTime = (new Date().getTime() - startTime) / 1000; this.speed = Math.round( (this.speed * this._progressEventCount + loaded / elapsedTime) / ++this._progressEventCount ); DynamicChunk.scale(this.speed); if (!throttle) { throttle = setTimeout(() => (throttle = undefined), 500); - const uploaded = (this.offset || 0) + loaded; + const uploaded = (this.offset as number) + loaded; this.progress = +((uploaded / this.size) * 100).toFixed(2); this.remaining = Math.ceil((this.size - uploaded) / this.speed); this.stateChange(this); diff --git a/src/uploadx/lib/utils.spec.ts b/src/uploadx/lib/utils.spec.ts index 7d34b03f..b020ef2d 100644 --- a/src/uploadx/lib/utils.spec.ts +++ b/src/uploadx/lib/utils.spec.ts @@ -1,4 +1,4 @@ -import { b64, DynamicChunk, isNumber, resolveUrl, unfunc } from './utils'; +import { b64, isNumber, resolveUrl, unfunc } from './utils'; /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -47,18 +47,3 @@ describe('primitives', () => { expect(unfunc((x: number) => 10 * x, 2)).toBe(20); }); }); - -describe('DynamicChunk', () => { - const init = DynamicChunk.size; - it('scale', () => { - expect(DynamicChunk.scale(0)).toEqual(init / 2); - expect(DynamicChunk.scale(0)).toEqual(init / 4); - expect(DynamicChunk.scale(Number.MAX_SAFE_INTEGER)).toEqual(init / 2); - expect(DynamicChunk.scale(Number.MAX_SAFE_INTEGER)).toEqual(init); - expect(DynamicChunk.scale(Number.MAX_SAFE_INTEGER)).toEqual(init * 2); - expect(DynamicChunk.scale(undefined as any)).toEqual(init * 2); - }); - afterEach(() => { - DynamicChunk.size = init; - }); -}); diff --git a/src/uploadx/lib/utils.ts b/src/uploadx/lib/utils.ts index 9ca25fa0..863fe461 100644 --- a/src/uploadx/lib/utils.ts +++ b/src/uploadx/lib/utils.ts @@ -75,31 +75,6 @@ export const b64 = { } }; -/** - * Adaptive chunk size - */ -export class DynamicChunk { - /** Maximum chunk size in bytes */ - static maxSize = Number.MAX_SAFE_INTEGER; - /** Minimum chunk size in bytes */ - static minSize = 1024 * 256; - /** Initial chunk size in bytes */ - static size = 4096 * 256; - static minChunkTime = 2; - static maxChunkTime = 8; - - static scale(throughput: number): number { - const elapsedTime = DynamicChunk.size / throughput; - if (elapsedTime < DynamicChunk.minChunkTime) { - DynamicChunk.size = Math.min(DynamicChunk.maxSize, DynamicChunk.size * 2); - } - if (elapsedTime > DynamicChunk.maxChunkTime) { - DynamicChunk.size = Math.max(DynamicChunk.minSize, DynamicChunk.size / 2); - } - return DynamicChunk.size; - } -} - export function isIOS(): boolean { return /iPad|iPhone|iPod/.test(navigator.platform) ? true