diff --git a/.gitignore b/.gitignore index b7dab5e9..4e6c00db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules -build \ No newline at end of file +build +json-schema-cache \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d3ab1329..1ce55934 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,7 @@ ADD package.json package-lock.json /app/ RUN npm install --omit=dev ADD build/src /app/ +# Cache of copy of the STAC JSON schemas by triggering a validation run +RUN node /app/index.js stac-validate https://nz-imagery.s3-ap-southeast-2.amazonaws.com/new-zealand/new-zealand_2020-2021_10m/rgb/2193/collection.json + ENTRYPOINT ["node", "/app/index.js"] diff --git a/src/commands/stac-validate/stac.validate.ts b/src/commands/stac-validate/stac.validate.ts index 60d85e90..fb73e0c1 100644 --- a/src/commands/stac-validate/stac.validate.ts +++ b/src/commands/stac-validate/stac.validate.ts @@ -2,6 +2,7 @@ import { fsa } from '@chunkd/fs'; import Ajv, { DefinedError, SchemaObject, ValidateFunction } from 'ajv'; import { fastFormats } from 'ajv-formats/dist/formats.js'; import { boolean, command, flag, number, option, restPositionals, string } from 'cmd-ts'; +import { createHash } from 'crypto'; import { dirname, join } from 'path'; import { performance } from 'perf_hooks'; import * as st from 'stac-ts'; @@ -13,6 +14,27 @@ import { hashStream } from '../../utils/hash.js'; import { Sha256Prefix } from '../../utils/hash.js'; import { config, registerCli, verbose } from '../common.js'; +/** + * Store a local copy of JSON schemas into a cache directory + * + * This is to prevent overloading the remote hosts as stac validation can trigger lots of schema requests + * + * @param url JSON schema to load + * @returns object from the cache if it exists or directly from the uri + */ +async function readSchema(url: string): Promise { + const cacheId = createHash('sha256').update(url).digest('hex'); + const cachePath = `./json-schema-cache/${cacheId}.json`; + try { + return await fsa.readJson(cachePath); + } catch (e) { + return fsa.readJson(url).then(async (obj) => { + await fsa.write(cachePath, JSON.stringify(obj)); + return obj; + }); + } +} + export const commandStacValidate = command({ name: 'stac-validate', description: 'Validate STAC files', @@ -78,8 +100,9 @@ export const commandStacValidate = command({ strict: args.strict, loadSchema: (uri: string): Promise => { let existing = Schemas.get(uri); + if (existing == null) { - existing = fsa.readJson(uri); + existing = readSchema(uri); Schemas.set(uri, existing); } return existing; @@ -99,7 +122,7 @@ export const commandStacValidate = command({ if (schema != null) return schema; let existing = ajvSchema.get(uri); if (existing == null) { - existing = fsa.readJson(uri).then((json) => ajv.compileAsync(json)); + existing = readSchema(uri).then((json) => ajv.compileAsync(json)); ajvSchema.set(uri, existing); } return existing;