Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: detects and error if the tiff is not 8 bits TDE-895 #890

Merged
merged 8 commits into from
Mar 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"@basemaps/geo": "^6.44.0",
"@chunkd/fs": "^10.0.9",
"@chunkd/source-aws-v3": "^10.1.3",
"@cogeotiff/core": "^8.1.1",
"@cogeotiff/core": "^9.0.3",
"@linzjs/geojson": "^6.43.0",
"@octokit/core": "^5.0.0",
"@octokit/plugin-rest-endpoint-methods": "^10.1.1",
Expand Down
6 changes: 3 additions & 3 deletions src/commands/common.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { fsa } from '@chunkd/fs';
import { CogTiff } from '@cogeotiff/core';
import { Tiff } from '@cogeotiff/core';
import { boolean, flag, option, optional, string } from 'cmd-ts';
import pLimit from 'p-limit';
import { fileURLToPath, pathToFileURL } from 'url';
Expand Down Expand Up @@ -91,10 +91,10 @@ const TiffQueue = pLimit(25);
* @param loc location to load the tiff from
* @returns Initialized tiff
*/
export function createTiff(loc: string): Promise<CogTiff> {
export function createTiff(loc: string): Promise<Tiff> {
const source = fsa.source(loc);

const tiff = new CogTiff({
const tiff = new Tiff({
url: tryParseUrl(loc),
fetch: (offset, length): Promise<ArrayBuffer> => {
/** Limit fetches concurrency see {@link TiffQueue} **/
Expand Down
10 changes: 5 additions & 5 deletions src/commands/path/path.generate.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Epsg } from '@basemaps/geo';
import { fsa } from '@chunkd/fs';
import { CogTiff } from '@cogeotiff/core';
import { Tiff } from '@cogeotiff/core';
import { command, option, positional, string } from 'cmd-ts';
import { StacCollection, StacItem } from 'stac-ts';

Expand Down Expand Up @@ -154,9 +154,9 @@ export function formatDate(collection: StacCollection): string {
* @async
* @param {string} source
* @param {StacCollection} collection
* @returns {Promise<CogTiff>}
* @returns {Promise<Tiff>}
*/
export async function loadFirstTiff(source: string, collection: StacCollection): Promise<CogTiff> {
export async function loadFirstTiff(source: string, collection: StacCollection): Promise<Tiff> {
const itemLink = collection.links.find((f) => f.rel === 'item')?.href;
if (itemLink == null) throw new Error(`No items in collection from ${source}.`);
const itemPath = new URL(itemLink, source).href;
Expand All @@ -170,15 +170,15 @@ export async function loadFirstTiff(source: string, collection: StacCollection):
return tiff;
}

export function extractGsd(tiff: CogTiff): number {
export function extractGsd(tiff: Tiff): number {
const gsd = tiff.images[0]?.resolution[0];
if (gsd == null) {
throw new Error(`Missing resolution tiff tag: ${tiff.source.url}`);
}
return gsd;
}

export function extractEpsg(tiff: CogTiff): number {
export function extractEpsg(tiff: Tiff): number {
const epsg = tiff.images[0]?.epsg;
if (epsg == null) {
throw new Error(`Missing epsg tiff tag: ${tiff.source.url}`);
Expand Down
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { CogTiff, CogTiffImage, Size } from '@cogeotiff/core';
import { Size, Tiff, TiffImage } from '@cogeotiff/core';

import { MapSheet } from '../../../utils/mapsheet.js';

Expand All @@ -16,12 +16,12 @@ const DefaultTiffImage = {
isGeoLocated: true,
};

export interface FakeCogTiffImage extends CogTiffImage {
export interface FakeCogTiffImage extends TiffImage {
epsg: number;
origin: [number, number, number];
}

export class FakeCogTiff extends CogTiff {
export class FakeCogTiff extends Tiff {
override images: [FakeCogTiffImage, ...FakeCogTiffImage[]];

constructor(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import { FeatureCollection } from 'geojson';

import { MapSheetData } from '../../../utils/__test__/mapsheet.data.js';
import { GridSize, MapSheet } from '../../../utils/mapsheet.js';
import { createTiff } from '../../common.js';
import {
commandTileIndexValidate,
extractTiffLocations,
getTileName,
GridSizeFromString,
groupByTileName,
TiffLoader,
validate8BitsTiff,
} from '../tileindex.validate.js';
import { FakeCogTiff } from './tileindex.validate.data.js';

Expand Down Expand Up @@ -276,3 +278,17 @@ describe('GridSizeFromString', () => {
);
});
});

describe('is8BitsTiff', () => {
it('should be a 8 bits TIFF', async () => {
const testTiff = await createTiff('./src/commands/tileindex-validate/__test__/data/8b.tiff');
await assert.doesNotReject(validate8BitsTiff(testTiff));
});
it('should not be a 8 bits TIFF', async () => {
const testTiff = await createTiff('./src/commands/tileindex-validate/__test__/data/16b.tiff');
await assert.rejects(validate8BitsTiff(testTiff), {
name: 'Error',
message: `${testTiff.source.url} is not a 8 bits TIFF`,
});
});
});
27 changes: 24 additions & 3 deletions src/commands/tileindex-validate/tileindex.validate.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Bounds, Projection } from '@basemaps/geo';
import { fsa } from '@chunkd/fs';
import { CogTiff, Size } from '@cogeotiff/core';
import { Size, Tiff, TiffTag } from '@cogeotiff/core';
import { boolean, command, flag, number, option, optional, restPositionals, string, Type } from 'cmd-ts';

import { CliInfo } from '../../cli.info.js';
Expand All @@ -25,7 +25,7 @@ export const TiffLoader = {
* @param args filter the tiffs
* @returns Initialized tiff
*/
async load(locations: string[], args?: FileFilter): Promise<CogTiff[]> {
async load(locations: string[], args?: FileFilter): Promise<Tiff[]> {
const files = await getFiles(locations, args);
const tiffLocations = files.flat().filter(isTiff);
if (tiffLocations.length === 0) throw new Error('No Files found');
Expand All @@ -46,6 +46,8 @@ export const TiffLoader = {
for (const prom of promises) {
// All the errors are logged above so just throw the first error
if (prom.status === 'rejected') throw new Error('Tiff loading failed: ' + String(prom.reason));
// We are processing only 8 bits Tiff for now
await validate8BitsTiff(prom.value);
output.push(prom.value);
}
return output;
Expand Down Expand Up @@ -286,7 +288,7 @@ export interface TiffLocation {
* @returns {TiffLocation[]}
*/
export async function extractTiffLocations(
tiffs: CogTiff[],
tiffs: Tiff[],
gridSize: GridSize,
forceSourceEpsg?: number,
): Promise<TiffLocation[]> {
Expand Down Expand Up @@ -416,3 +418,22 @@ export function getTileName(x: number, y: number, gridSize: GridSize): string {
const tileId = `${`${tileY}`.padStart(nbDigits, '0')}${`${tileX}`.padStart(nbDigits, '0')}`;
return `${sheetCode}_${gridSize}_${tileId}`;
}

/**
* Validate if a TIFF contains only 8 bits bands.
*
* @param tiff
*/
export async function validate8BitsTiff(tiff: Tiff): Promise<void> {
const baseImage = tiff.images[0];
if (baseImage === undefined) throw new Error(`Can't get base image for ${tiff.source.url}`);

const bitsPerSample = await baseImage.fetch(TiffTag.BitsPerSample);
if (bitsPerSample == null) {
throw new Error(`Failed to extract band information from ${tiff.source.url}`);
}

if (!bitsPerSample.every((currentNumberBits) => currentNumberBits === 8)) {
throw new Error(`${tiff.source.url} is not a 8 bits TIFF`);
}
}
16 changes: 8 additions & 8 deletions src/utils/__test__/geotiff.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { describe, it } from 'node:test';

import { fsa } from '@chunkd/fs';
import { FsMemory } from '@chunkd/source-memory';
import { CogTiff, CogTiffImage, Source } from '@cogeotiff/core';
import { Source, Tiff, TiffImage } from '@cogeotiff/core';

import { createTiff } from '../../commands/common.js';
import { findBoundingBox, parseTfw, PixelIsPoint } from '../geotiff.js';
Expand Down Expand Up @@ -77,7 +77,7 @@ describe('geotiff', () => {
const fakeSource: Source = { url: url, fetch: async () => new ArrayBuffer(1) };
it('should not parse a tiff with no information ', async () => {
// tiff with no location information and no TFW
await assert.rejects(() => findBoundingBox({ source: fakeSource, images: [] } as unknown as CogTiff));
await assert.rejects(() => findBoundingBox({ source: fakeSource, images: [] } as unknown as Tiff));
});

it('should parse a tiff with TFW', async () => {
Expand All @@ -86,8 +86,8 @@ describe('geotiff', () => {
// tiff with no location information and no TFW
const bbox = await findBoundingBox({
source: fakeSource,
images: [{ size: { width: 3200, height: 4800 } }] as unknown as CogTiffImage,
} as unknown as CogTiff);
images: [{ size: { width: 3200, height: 4800 } }] as unknown as TiffImage,
} as unknown as Tiff);
assert.deepEqual(bbox, [1460800, 5079120, 1461040, 5079480]);
await fsa.delete('memory://BX20_500_023098.tfw');
});
Expand All @@ -104,9 +104,9 @@ describe('geotiff', () => {
valueGeo(): number {
return 1; // PixelIsArea
},
} as unknown as CogTiffImage,
} as unknown as TiffImage,
],
} as unknown as CogTiff);
} as unknown as Tiff);
assert.deepEqual(bbox, [1460800, 5079120, 1461040, 5079480]);
});
it('should parse with pixel offset', async () => {
Expand All @@ -121,9 +121,9 @@ describe('geotiff', () => {
valueGeo(): number {
return PixelIsPoint;
},
} as unknown as CogTiffImage,
} as unknown as TiffImage,
],
} as unknown as CogTiff);
} as unknown as Tiff);
assert.deepEqual(bbox, [1460800, 5079120, 1461040, 5079480]);
});
});
4 changes: 2 additions & 2 deletions src/utils/geotiff.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { fsa } from '@chunkd/fs';
import { CogTiff, TiffTagGeo } from '@cogeotiff/core';
import { Tiff, TiffTagGeo } from '@cogeotiff/core';

import { urlToString } from '../commands/common.js';

Expand Down Expand Up @@ -52,7 +52,7 @@ export const PixelIsPoint = 2;
*
* @returns [minX, minY, maxX, maxY] bounding box
*/
export async function findBoundingBox(tiff: CogTiff): Promise<[number, number, number, number]> {
export async function findBoundingBox(tiff: Tiff): Promise<[number, number, number, number]> {
const img = tiff.images[0];
if (img == null) throw new Error(`Failed to find bounding box/origin - no images found in file: ${tiff.source.url}`);
const size = img.size;
Expand Down
Loading