THR-7 ContentStorage: store data on S3 via S3ClientAdapter

apps/server/src/modules/files-storage/client/s3-client.adapter.spec.ts

@@ -430,4 +430,152 @@ describe('S3ClientAdapter', () => {
 			await expect(service.copy(undefined)).rejects.toThrowError(InternalServerErrorException);
 		});
 	});
+
+	describe('head', () => {
+		const setup = () => {
+			const { pathToFile } = createParameter();
+
+			return { pathToFile };
+		};
+
+		describe('when file exists', () => {
+			it('should call send() of client with head object', async () => {
+				const { pathToFile } = setup();
+
+				await service.head(pathToFile);
+
+				expect(client.send).toBeCalledWith(
+					expect.objectContaining({
+						input: { Bucket: 'test-bucket', Key: pathToFile },
+					})
+				);
+			});
+		});
+
+		describe('when file does not exist', () => {
+			it('should throw NotFoundException', async () => {
+				const { pathToFile } = setup();
+				// @ts-expect-error ignore parameter type of mock function
+				client.send.mockRejectedValueOnce(new Error('NoSuchKey'));
+
+				const headPromise = service.head(pathToFile);
+
+				await expect(headPromise).rejects.toBeInstanceOf(NotFoundException);
+			});
+		});
+	});
+
+	describe('list', () => {
+		const setup = () => {
+			const prefix = 'test/';
+
+			const keys = Array.from(Array(2500).keys()).map((n) => `KEY-${n}`);
+			const responseContents = keys.map((key) => {
+				return {
+					Key: `${prefix}${key}`,
+				};
+			});
+
+			return { prefix, keys, responseContents };
+		};
+
+		afterEach(() => {
+			client.send.mockClear();
+		});
+
+		it('should truncate result when max is given', async () => {
+			const { prefix, keys, responseContents } = setup();
+
+			// @ts-expect-error ignore parameter type of mock function
+			client.send.mockResolvedValue({
+				IsTruncated: false,
+				Contents: responseContents.slice(0, 500),
+			});
+
+			const resultKeys = await service.list(prefix, 500);
+
+			expect(resultKeys).toEqual(keys.slice(0, 500));
+
+			expect(client.send).toBeCalledWith(
+				expect.objectContaining({
+					input: {
+						Bucket: 'test-bucket',
+						Prefix: prefix,
+						ContinuationToken: undefined,
+						MaxKeys: 500,
+					},
+				})
+			);
+		});
+
+		it('should call send() multiple times if bucket contains more than 1000 keys', async () => {
+			const { prefix, responseContents, keys } = setup();
+
+			client.send
+				// @ts-expect-error ignore parameter type of mock function
+				.mockResolvedValueOnce({
+					IsTruncated: true,
+					NextContinuationToken: '1',
+					Contents: responseContents.slice(0, 1000),
+				})
+				// @ts-expect-error ignore parameter type of mock function
+				.mockResolvedValueOnce({
+					IsTruncated: true,
+					NextContinuationToken: '2',
+					Contents: responseContents.slice(1000, 2000),
+				})
+				// @ts-expect-error ignore parameter type of mock function
+				.mockResolvedValueOnce({
+					Contents: responseContents.slice(2000),
+				});
+
+			const resultKeys = await service.list(prefix);
+
+			expect(resultKeys).toEqual(keys);
+
+			expect(client.send).toHaveBeenNthCalledWith(
+				1,
+				expect.objectContaining({
+					input: {
+						Bucket: 'test-bucket',
+						Prefix: prefix,
+						ContinuationToken: undefined,
+					},
+				})
+			);
+
+			expect(client.send).toHaveBeenNthCalledWith(
+				2,
+				expect.objectContaining({
+					input: {
+						Bucket: 'test-bucket',
+						Prefix: prefix,
+						ContinuationToken: '1',
+					},
+				})
+			);
+
+			expect(client.send).toHaveBeenNthCalledWith(
+				3,
+				expect.objectContaining({
+					input: {
+						Bucket: 'test-bucket',
+						Prefix: prefix,
+						ContinuationToken: '2',
+					},
+				})
+			);
+		});
+
+		it('should throw error if client rejects with an error', async () => {
+			const { prefix } = setup();
+
+			// @ts-expect-error ignore parameter type of mock function
+			client.send.mockRejectedValue(new Error());
+
+			const listPromise = service.list(prefix);
+
+			await expect(listPromise).rejects.toThrow();
+		});
+	});
 });

apps/server/src/modules/files-storage/client/s3-client.adapter.ts

@@ -4,6 +4,9 @@ import {
 	CreateBucketCommand,
 	DeleteObjectsCommand,
 	GetObjectCommand,
+	HeadObjectCommand,
+	ListObjectsV2Command,
+	ListObjectsV2CommandOutput,
 	S3Client,
 	ServiceOutputTypes,
 } from '@aws-sdk/client-s3';
@@ -183,6 +186,61 @@ export class S3ClientAdapter implements IStorageClient {
 		return this.client.send(req);
 	}
 
+	public async list(prefix: string, maxKeys?: number) {
+		this.logger.log({ action: 'list', params: { prefix, bucket: this.config.bucket } });
+
+		try {
+			let files: string[] = [];
+			let ret: ListObjectsV2CommandOutput | undefined;
+
+			do {
+				const req = new ListObjectsV2Command({
+					Bucket: this.config.bucket,
+					Prefix: prefix,
+					ContinuationToken: ret?.NextContinuationToken,
+					MaxKeys: maxKeys && maxKeys - files.length,
+				});
+
+				// Iterations are dependent on each other
+				// eslint-disable-next-line no-await-in-loop
+				ret = await this.client.send(req);
+
+				const returnedFiles =
+					ret?.Contents?.filter((o) => o.Key)
+						.map((o) => o.Key as string) // Can not be undefined because of filter above
+						.map((key) => key.substring(prefix.length)) ?? [];
+
+				files = files.concat(returnedFiles);
+			} while (ret?.IsTruncated && (!maxKeys || files.length < maxKeys));
+
+			return files;
+		} catch (err) {
+			throw new InternalServerErrorException(err, 'S3ClientAdapter:list');
+		}
+	}
+
+	public async head(path: string) {
+		try {
+			this.logger.log({ action: 'head', params: { path, bucket: this.config.bucket } });
+
+			const req = new HeadObjectCommand({
+				Bucket: this.config.bucket,
+				Key: path,
+			});
+
+			const headResponse = await this.client.send(req);
+
+			return headResponse;
+		} catch (err) {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+			if (err.message && err.message === 'NoSuchKey') {
+				this.logger.log(`could not find the file for head with id ${path}`);
+				throw new NotFoundException('NoSuchKey');
+			}
+			throw new InternalServerErrorException(err, 'S3ClientAdapter:head');
+		}
+	}
+
 	/* istanbul ignore next */
 	private checkStreamResponsive(stream: Readable, context: string) {
 		let timer: NodeJS.Timeout;