feat(basemaps): Update the create-pr cli to add new individual vector tileset. BM-992

package.json

@@ -45,6 +45,7 @@
     "@linzjs/geojson": "^6.43.0",
     "@octokit/core": "^5.0.0",
     "@octokit/plugin-rest-endpoint-methods": "^10.1.1",
+    "@octokit/request-error": "^6.0.2",
     "@wtrpc/core": "^1.0.2",
     "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",

src/commands/basemaps-github/create-pr.ts

@@ -13,7 +13,15 @@ import { Category, MakeCogGithub } from './make.cog.github.js';
 const validTargetBuckets: Set<string> = new Set(['linz-basemaps', 'linz-basemaps-staging']);
 const validSourceBuckets: Set<string> = new Set(['nz-imagery', 'linz-imagery']);
 
-async function parseTargetInfo(
+function assertValidBucket(bucket: string, validBuckets: Set<string>): void {
+  // Validate the target information
+  logger.info({ bucket }, 'CreatePR: Valid the target s3 bucket');
+  if (!validBuckets.has(bucket)) {
+    throw new Error(`Invalid s3 bucket ${bucket} from the target.`);
+  }
+}
+
+async function parseRasterTargetInfo(
   target: string,
   individual: boolean,
 ): Promise<{ name: string; title: string; epsg: EpsgCode; region: string | undefined }> {
@@ -24,11 +32,7 @@ async function parseTargetInfo(
   const epsg = Epsg.tryGet(Number(splits[1]));
   const name = splits[2];
 
-  //Validate the target information
-  logger.info({ bucket }, 'CreatePR: Valid the target s3 bucket');
-  if (bucket == null || !validTargetBuckets.has(bucket)) {
-    throw new Error(`Invalid s3 bucket ${bucket} from the target ${target}.`);
-  }
+  assertValidBucket(bucket, validTargetBuckets);
 
   if (epsg == null || name == null) throw new Error(`Invalid target ${target} to parse the epsg and imagery name.`);
   const collectionPath = fsa.join(target, 'collection.json');
@@ -42,10 +46,8 @@ async function parseTargetInfo(
   if (source == null) throw new Error(`Failed to get source url from collection.json.`);
   const sourceUrl = new URL(source);
   const sourceBucket = sourceUrl.hostname;
-  logger.info({ bucket: sourceBucket }, 'CreatePR: Validate the source s3 bucket');
-  if (sourceBucket == null || !validSourceBuckets.has(sourceBucket)) {
-    throw new Error(`Invalid s3 bucket ${sourceBucket} from the source ${sourceUrl}.`);
-  }
+  assertValidBucket(sourceBucket, validSourceBuckets);
+
   // Try to get the region for individual layers
   let region;
   if (individual) {
@@ -61,6 +63,28 @@ async function parseTargetInfo(
   return { name: standardizeLayerName(name), epsg: epsg.code, title, region };
 }
 
+/**
+ * Pass vector target location with the following format
+ * s3://linz-basemaps-staging/vector/3857/53382-nz-roads-addressing/01HSF04SG9M1P3V667A4NZ1MN8/53382-nz-roads-addressing.tar.co
+ * s3://linz-basemaps-staging/vector/3857/topographic/01HSF04SG9M1P3V667A4NZ1MN8/topographic.tar.co
+ */
+async function parseVectorTargetInfo(target: string): Promise<{ name: string; title: string; epsg: EpsgCode }> {
+  logger.info({ target }, 'CreatePR: Get the layer information from target');
+  const url = new URL(target);
+  const bucket = url.hostname;
+  const splits = url.pathname.split('/');
+  const epsg = Epsg.tryGet(Number(splits[2]));
+  const name = splits[3];
+
+  assertValidBucket(bucket, validTargetBuckets);
+
+  if (epsg == null || name == null) throw new Error(`Invalid target ${target} to parse the epsg and imagery name.`);
+  if (epsg !== Epsg.Google) throw new Error(`Unsupported epsg code ${epsg.code} for vector map.`);
+  // Try to get the region for individual layers
+
+  return { name: name, epsg: epsg.code, title: name };
+}
+
 export const CommandCreatePRArgs = {
   verbose,
   target: option({
@@ -113,12 +137,15 @@ export const basemapsCreatePullRequest = command({
     const layer: ConfigLayer = { name: '', title: '' };
     let region;
     if (args.vector) {
-      layer.name = 'topographic';
-      layer.title = 'Topographic';
-      layer[3857] = targets[0];
+      for (const target of targets) {
+        const info = await parseVectorTargetInfo(target);
+        layer.name = info.name;
+        layer.title = info.title;
+        layer[info.epsg] = target;
+      }
     } else {
       for (const target of targets) {
-        const info = await parseTargetInfo(target, args.individual);
+        const info = await parseRasterTargetInfo(target, args.individual);
         layer.name = info.name;
         layer.title = info.title;
         layer[info.epsg] = target;
@@ -130,7 +157,7 @@ export const basemapsCreatePullRequest = command({
     if (layer.name === '' || layer.title === '') throw new Error('Failed to find the imagery name or title.');
 
     const git = new MakeCogGithub(layer.name, args.repository);
-    if (args.vector) await git.updateVectorTileSet('topographic', layer);
+    if (args.vector) await git.updateVectorTileSet(layer.name, layer);
     else await git.updateRasterTileSet('aerial', layer, category, region);
   },
 });

src/commands/basemaps-github/make.cog.github.ts

@@ -5,6 +5,7 @@ import {
   TileSetType,
 } from '@basemaps/config/build/config/tile.set.js';
 import { TileSetConfigSchema } from '@basemaps/config/build/json/parse.tile.set.js';
+import { VectorFormat } from '@basemaps/geo';
 import { fsa } from '@chunkd/fs';
 
 import { logger } from '../../log.js';
@@ -83,6 +84,7 @@ export class MakeCogGithub {
       // Prepare new aerial tileset config
       const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
       const tileSetContent = await gh.getContent(tileSetPath);
+      if (tileSetContent == null) throw new Error(`Unable get the ${filename}.json from config repo.`);
       const tileSet = JSON.parse(tileSetContent) as ConfigTileSetRaster;
       const newTileSet = await this.prepareRasterTileSetConfig(layer, tileSet, category);
       // skip pull request if not an urban or rural imagery
@@ -170,11 +172,14 @@ export class MakeCogGithub {
     logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file');
     const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`);
     const tileSetContent = await gh.getContent(tileSetPath);
-    const tileSet = JSON.parse(tileSetContent) as ConfigTileSetVector;
-    const newTileSet = await this.prepareVectorTileSetConfig(layer, tileSet);
 
-    // skip pull request if not an urban or rural imagery
-    if (newTileSet == null) return;
+    // update the existing tileset
+    const existingTileSet = tileSetContent != null ? (JSON.parse(tileSetContent) as ConfigTileSetVector) : undefined;
+    const newTileSet = await this.prepareVectorTileSetConfig(layer, existingTileSet);
+
+    // skip pull request tileset prepare failure.
+    if (newTileSet == null) throw new Error('Failed to prepare new Vector tileSet.');
+
     // Github
     const title = `config(vector): Update the ${this.imagery} to ${filename} config file.`;
     const content = await prettyPrint(JSON.stringify(newTileSet, null, 2), ConfigPrettierFormat);
@@ -186,7 +191,19 @@ export class MakeCogGithub {
   /**
    * Prepare raster tileSet config json
    */
-  async prepareVectorTileSetConfig(layer: ConfigLayer, tileSet: ConfigTileSetVector): Promise<ConfigTileSetVector> {
+  async prepareVectorTileSetConfig(layer: ConfigLayer, tileSet?: ConfigTileSetVector): Promise<ConfigTileSetVector> {
+    if (tileSet == null) {
+      return {
+        type: TileSetType.Vector,
+        id: `ts_${layer.name}`,
+        name: layer.name,
+        title: layer.title,
+        maxZoom: 15,
+        format: VectorFormat.MapboxVectorTiles,
+        layers: [layer],
+      };
+    }
+
     // Reprocess existing layer
     for (let i = 0; i < tileSet.layers.length; i++) {
       if (tileSet.layers[i]?.name === layer.name) {

src/commands/stac-github-import/stac.github.import.ts

@@ -91,6 +91,7 @@ export const commandStacGithubImport = command({
     logger.info({ template: fsa.joinAll('template', 'catalog.json') }, 'Stac:ReadTemplate');
     const catalogPath = fsa.joinAll('template', 'catalog.json');
     const catalog = await gh.getContent(catalogPath);
+    if (catalog == null) throw new Error(`Failed to get catalog.json from ${args.repoName} repo.`);
     const catalogJson = JSON.parse(catalog) as st.StacCatalog;
 
     // Catalog template should have a absolute link to itself

src/utils/github.ts

@@ -1,6 +1,7 @@
 import { Octokit } from '@octokit/core';
 import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods';
 import { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js';
+import { RequestError } from '@octokit/request-error';
 
 import { logger } from '../log.js';
 
@@ -94,15 +95,26 @@ export class GithubApi {
   /**
    * Get content from the github repository
    */
-  async getContent(path: string): Promise<string> {
+  async getContent(path: string): Promise<string | null> {
     logger.info({ path }, 'GitHub API: Get Content');
-    const response = await this.octokit.rest.repos.getContent({ owner: this.owner, repo: this.repo, path });
-    if (!this.isOk(response.status)) throw new Error('Failed to get aerial TileSet config.');
-    if ('content' in response.data) {
-      return Buffer.from(response.data.content, 'base64').toString();
-    } else {
-      throw new Error(`Unable to find the content from path ${path}.`);
+    try {
+      const response = await this.octokit.rest.repos.getContent({ owner: this.owner, repo: this.repo, path });
+      if (this.isOk(response.status) && 'content' in response.data) {
+        return Buffer.from(response.data.content, 'base64').toString();
+      } else {
+        throw new Error('GitHub: getContent return no content in response data.');
+      }
+    } catch (error) {
+      // Trying to catch the non found response and return null
+      if (error instanceof RequestError && error.status === 404) {
+        logger.info({ path }, 'GitHub API: Content Not Found');
+        return null;
+      } else {
+        throw error;
+      }
     }
+
+    throw new Error('GitHub: Get content Failure');
   }
 
   /**