feat(cogify): force fully qualified domain names for s3 to reduce DNS load TDE-1084

packages/cogify/src/bin.ts

@@ -1,11 +1,13 @@
 Error.stackTraceLimit = 100;
 
-import { LogConfig } from '@basemaps/shared';
-import { fsa } from '@basemaps/shared';
+import { Fqdn, fsa, LogConfig } from '@basemaps/shared';
 import { run } from 'cmd-ts';
 
 import { CogifyCli } from './cogify/cli.js';
 
+// Force Fully qualified domain names to reduce DNS lookups
+Fqdn.isForcedFqdn = true;
+
 // remove the source caching / chunking as it is not needed for cogify, cogify only reads tiffs once so caching the result is not helpful
 fsa.middleware = fsa.middleware.filter((f) => f.name !== 'source:chunk');
 fsa.middleware = fsa.middleware.filter((f) => f.name !== 'source:cache');

packages/shared/src/__tests__/fqdn.test.ts

@@ -0,0 +1,48 @@
+import assert from 'node:assert';
+import { beforeEach, describe, it } from 'node:test';
+
+import { FinalizeHandler, MetadataBearer } from '@smithy/types';
+
+import { Fqdn } from '../file.system.middleware.js';
+
+describe('fqdnMiddleware', () => {
+  const fakeNext: FinalizeHandler<object, MetadataBearer> = () => {
+    return Promise.resolve({ output: { $metadata: {} }, response: {} });
+  };
+  const fakeRequest = { input: {}, request: { hostname: 'nz-imagery.s3.ap-southeast-2.amazonaws.com' } };
+
+  beforeEach(() => {
+    Fqdn.isForcedFqdn = true;
+  });
+
+  it('should not add  FQDN to s3 requests if turned off', () => {
+    Fqdn.isForcedFqdn = false;
+    fakeRequest.request.hostname = 'nz-imagery.s3.ap-southeast-2.amazonaws.com';
+    Fqdn.middleware(fakeNext, {})(fakeRequest);
+    assert.equal(fakeRequest.request.hostname, 'nz-imagery.s3.ap-southeast-2.amazonaws.com');
+  });
+
+  it('should add FQDN to s3 requests', () => {
+    fakeRequest.request.hostname = 'nz-imagery.s3.ap-southeast-2.amazonaws.com';
+    Fqdn.middleware(fakeNext, {})(fakeRequest);
+    assert.equal(fakeRequest.request.hostname, 'nz-imagery.s3.ap-southeast-2.amazonaws.com.');
+  });
+
+  it('should not add for other services', () => {
+    fakeRequest.request.hostname = 'logs.ap-southeast-2.amazonaws.com';
+    Fqdn.middleware(fakeNext, {})(fakeRequest);
+    assert.equal(fakeRequest.request.hostname, 'logs.ap-southeast-2.amazonaws.com');
+  });
+
+  it('should not add for other regions', () => {
+    fakeRequest.request.hostname = 'nz-imagery.s3.us-east-1.amazonaws.com';
+    Fqdn.middleware(fakeNext, {})(fakeRequest);
+    assert.equal(fakeRequest.request.hostname, 'nz-imagery.s3.us-east-1.amazonaws.com');
+  });
+
+  it('should not add for unknown hosts', () => {
+    fakeRequest.request.hostname = 'google.com';
+    Fqdn.middleware(fakeNext, {})(fakeRequest);
+    assert.equal(fakeRequest.request.hostname, 'google.com');
+  });
+});

packages/shared/src/file.system.middleware.ts

@@ -0,0 +1,40 @@
+import { FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types';
+
+/** Force fully qualifed domain names (FQDN) for s3 requests to save DNS lookups */
+interface Fqdn {
+  /**
+   * Should S3 requests be converted to Fully Qualified domains (FQDN)
+   *
+   * @default false
+   */
+  isForcedFqdn: boolean;
+  /**
+   * AWS SDK middleware function to force fully qualified domain name  on s3 requests
+   *
+   * AWS S3 inside of kubernetes triggers a lot of DNS requests
+   * by forcing a fully qualified domain name lookup (trailing ".")
+   * it greatly reduces the number of DNS requests we make
+   */
+  middleware: FinalizeRequestMiddleware<object, MetadataBearer>;
+}
+export const Fqdn: Fqdn = {
+  isForcedFqdn: false,
+  middleware: (next) => {
+    return (args) => {
+      // Forced FQDN is disabled
+      if (Fqdn.isForcedFqdn === false) return next(args);
+
+      if (hasHostName(args.request) && args.request.hostname.endsWith('.s3.ap-southeast-2.amazonaws.com')) {
+        args.request.hostname += '.';
+      }
+      return next(args);
+    };
+  },
+};
+
+/** Check to see if hostname exists inside of a object */
+function hasHostName(x: unknown): x is { hostname: string } {
+  if (x == null) return false;
+  if (typeof x === 'object' && 'hostname' in x && typeof x.hostname === 'string') return true;
+  return false;
+}

packages/shared/src/file.system.ts

@@ -9,6 +9,7 @@ import type { SourceCallback, SourceRequest } from '@chunkd/source';
 import type { RequestSigner } from '@smithy/types';
 
 import { Env } from './const.js';
+import { Fqdn } from './file.system.middleware.js';
 import { LogConfig } from './log.js';
 
 export const s3Fs = new FsAwsS3(
@@ -32,10 +33,20 @@ export const s3FsPublic = new FsAwsS3(
   }),
 );
 
+/** Ensure middleware are added to all s3 clients that are created */
+function applyS3MiddleWare(fs: FsAwsS3): void {
+  if (fs.s3 == null) return;
+  fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
+}
+
+applyS3MiddleWare(s3FsPublic);
+applyS3MiddleWare(s3Fs);
+
 const credentials = new AwsS3CredentialProvider();
 
 credentials.onFileSystemCreated = (acc: AwsCredentialConfig, fs: FileSystem): void => {
   LogConfig.get().debug({ prefix: acc.prefix, roleArn: acc.roleArn }, 'FileSystem:Register');
+  applyS3MiddleWare(fs as FsAwsS3);
   fsa.register(acc.prefix, fs);
 };
 

packages/shared/src/index.ts

@@ -6,6 +6,7 @@ export { Const, Env } from './const.js';
 export { ConfigDynamoBase } from './dynamo/dynamo.config.base.js';
 export { ConfigProviderDynamo } from './dynamo/dynamo.config.js';
 export { Fsa as fsa, FsaCache, FsaChunk, FsaLog, stringToUrlFolder, urlToString } from './file.system.js';
+export { Fqdn } from './file.system.middleware.js';
 export { getImageryCenterZoom, getPreviewQuery, getPreviewUrl, PreviewSize } from './imagery.url.js';
 export { LogConfig, LogType } from './log.js';
 export { LoggerFatalError } from './logger.fatal.error.js';