Skip to content

Commit

Permalink
fix: validate existing files on create
Browse files Browse the repository at this point in the history
  • Loading branch information
@kukhariev
kukhariev committed Feb 6, 2020
1 parent 19becba commit 5519888
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 45 deletions.
39 changes: 24 additions & 15 deletions src/storages/gcs-storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,11 @@ export class GCStorage extends BaseStorage<GCSFile, CGSObject> {
const file = new GCSFile(config);
await this.validate(file);
file.name = this.namingFunction(file);
const existing = await this._getMeta(file.name).catch(noop);
if (existing) return existing;

try {
const existing = await this._getMeta(file.name);
existing.bytesWritten = await this._write(existing);
if (existing.bytesWritten >= 0) return existing;
} catch {}
const origin = getHeader(req, 'origin');
const headers = { 'Content-Type': 'application/json; charset=utf-8' } as any;
headers['X-Upload-Content-Length'] = file.size.toString();
Expand Down Expand Up @@ -180,35 +182,42 @@ export class GCStorage extends BaseStorage<GCSFile, CGSObject> {
}

private async _saveMeta(file: GCSFile): Promise<any> {
const name = file.name;
await this.authClient.request({
body: JSON.stringify(file),
headers: { 'Content-Type': 'application/json; charset=utf-8' },
method: 'POST',
params: { name: `${encodeURIComponent(file.name)}${METAFILE_EXTNAME}`, uploadType: 'media' },
params: { name: this.metaName(name), uploadType: 'media' },
url: this.uploadBaseURI
});
this.cache[file.name] = file;
this.cache[name] = file;
return file;
}

private async _getMeta(name: string): Promise<GCSFile> {
const file = this.cache[name];
if (file) return file;
try {
const url = `${this.storageBaseURI}/${encodeURIComponent(name)}${METAFILE_EXTNAME}`;
const url = `${this.storageBaseURI}/${this.metaName(name)}`;
const { data } = await this.authClient.request<GCSFile>({ params: { alt: 'media' }, url });
if (data.name !== name) return fail(ERRORS.FILE_NOT_FOUND);
this.cache[name] = data;
return data;
} catch (error) {
return fail(ERRORS.FILE_NOT_FOUND, error);
}
if (data?.name === name) {
this.cache[name] = data;
return data;
}
} catch (error) {}
return fail(ERRORS.FILE_NOT_FOUND);
}

private async _deleteMeta(name: string): Promise<any> {
const url = `${this.storageBaseURI}/${encodeURIComponent(name)}${METAFILE_EXTNAME}`;
private async _deleteMeta(name: string): Promise<void> {
const url = `${this.storageBaseURI}/${this.metaName(name)}`;
try {
await this.authClient.request({ method: 'DELETE', url });
} catch {}
delete this.cache[name];
return this.authClient.request({ method: 'DELETE', url }).catch(err => this.log(err));
}

private metaName(name: string): string {
return `${encodeURIComponent(name)}${METAFILE_EXTNAME}`;
}

private async _checkBucket(bucketName: string): Promise<any> {
Expand Down
47 changes: 25 additions & 22 deletions src/storages/s3-storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const BUCKET_NAME = 'node-uploadx';
export class S3File extends File {
Parts: S3.Parts = [];
UploadId = '';
uri = '';
uri?: string;
}

export type S3StorageOptions = BaseStorageOptions &
Expand Down Expand Up @@ -55,8 +55,11 @@ export class S3Storage extends BaseStorage<S3File, any> {
const file = new S3File(config);
await this.validate(file);
file.name = this.namingFunction(file);
const existing = await this._getMeta(file.name).catch(noop);
if (existing) return existing;
try {
const existing = await this._getMeta(file.name);
existing.bytesWritten = await this._write(existing);
if (existing.bytesWritten >= 0) return existing;
} catch {}
const metadata = processMetadata(file.metadata, encodeURI);

const multiPartOptions: S3.CreateMultipartUploadRequest = {
Expand All @@ -78,7 +81,6 @@ export class S3Storage extends BaseStorage<S3File, any> {

async write(part: FilePart): Promise<S3File> {
const file = await this._getMeta(part.name);
if (!file) return fail(ERRORS.FILE_NOT_FOUND);
file.bytesWritten = await this._write({ ...file, ...part });
file.status = this.setStatus(file);
if (file.status === 'completed') {
Expand Down Expand Up @@ -180,27 +182,28 @@ export class S3Storage extends BaseStorage<S3File, any> {
const { Metadata } = await this.client
.headObject({ Bucket: this.bucket, Key: name + METAFILE_EXTNAME })
.promise();
if (!Metadata) return fail(ERRORS.FILE_NOT_FOUND);
file = JSON.parse(decodeURIComponent(Metadata.metadata));
const { Parts } = await this._listParts(name, file?.UploadId);
file.Parts = Parts || [];
file.bytesWritten = file.Parts.map(item => item.Size || 0).reduce(
(prev, next) => prev + next,
0
);
this.cache[name] = file;
return file;
} catch (e) {
return fail(ERRORS.FILE_NOT_FOUND, e);
}
if (Metadata) {
file = JSON.parse(decodeURIComponent(Metadata.metadata));
const { Parts } = await this._listParts(name, file?.UploadId);
file.Parts = Parts || [];
file.bytesWritten = file.Parts.map(item => item.Size || 0).reduce(
(prev, next) => prev + next,
0
);
this.cache[name] = file;
return file;
}
} catch (e) {}
return fail(ERRORS.FILE_NOT_FOUND);
}

private async _deleteMeta(file: S3File): Promise<any> {
private async _deleteMeta(file: S3File): Promise<void> {
delete this.cache[file.name];
await this.client
.deleteObject({ Bucket: this.bucket, Key: file.name + METAFILE_EXTNAME })
.promise()
.catch(noop);
try {
await this.client
.deleteObject({ Bucket: this.bucket, Key: file.name + METAFILE_EXTNAME })
.promise();
} catch {}
}

private async _abortMultipartUpload(file: S3File): Promise<any> {
Expand Down
24 changes: 17 additions & 7 deletions test/gcs-storage.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ describe('GCStorage', () => {
const req = { headers: { origin: 'http://api.com' } } as any;

beforeEach(async () => {
mockAuthRequest.mockResolvedValue({ buket: 'ok' });
mockAuthRequest.mockResolvedValueOnce({ buket: 'ok' });
storage = new GCStorage({ ...storageOptions });
file = _fileResponse().data;
});
Expand All @@ -32,18 +32,27 @@ describe('GCStorage', () => {

describe('.create()', () => {
it('should request api and set status and uri', async () => {
mockAuthRequest
.mockRejectedValueOnce({ code: 404 })
.mockResolvedValueOnce(_createResponse())
.mockResolvedValue({});
mockAuthRequest.mockRejectedValueOnce({ code: 404, detail: 'meta not found' });
mockAuthRequest.mockResolvedValueOnce(_createResponse());
mockAuthRequest.mockResolvedValueOnce('_saveOk');
file = await storage.create(req, testfile);
expect(file.name).toEqual(filename);
expect(file.status).toEqual('created');
expect(file).toMatchObject({ ...testfile, uri });
expect(mockAuthRequest).toHaveBeenCalledTimes(4);
expect(mockAuthRequest).toBeCalledWith(request.create);
const existing = await storage.create(req, testfile);
expect(file).toMatchObject(existing);
});

it('should return existing file', async () => {
mockFetch.mockResolvedValueOnce(
new Response('', {
status: 308,
headers: { Range: '0-5' }
})
);
storage['cache'] = { [filename]: { ...testfile, uri } };
file = await storage.create(req, testfile);
expect(file.uri).toEqual(uri);
});

it('should reject on api error', async () => {
Expand Down Expand Up @@ -83,6 +92,7 @@ describe('GCStorage', () => {
describe('.write()', () => {
it('should request api and set status and bytesWritten', async () => {
mockAuthRequest.mockResolvedValueOnce(_fileResponse());
mockAuthRequest.mockResolvedValueOnce('_delete');
mockFetch.mockResolvedValueOnce(new Response('{"mediaLink":"http://api.com/123456789"}'));
const body = createReadStream(srcpath);
const part: FilePart = {
Expand Down
6 changes: 5 additions & 1 deletion test/s3-storage.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,12 @@ describe('S3Storage', () => {
UploadId: expect.any(String),
Parts: expect.any(Array)
});
});

it('should return existing file', async () => {
storage['cache'] = { [filename]: { ...file } };
const existing = await storage.create({} as any, testfile);
expect(file).toMatchObject(existing);
expect(existing.UploadId).toEqual(file.UploadId);
});
});
describe('.update()', () => {
Expand Down

0 comments on commit 5519888

Please sign in to comment.