Skip to content

Commit

Permalink
Merge pull request #25 from koopjs/f/3978-customizable-spatial-attribute
Browse files Browse the repository at this point in the history 
F/3978 customizable spatial attribute
  • Loading branch information
@drspacemanphd
drspacemanphd committed Jun 24, 2022
2 parents 3b33c03 + 62e22e2 commit 21189e3
Show file tree
Hide file tree
Showing 5 changed files with 323 additions and 27 deletions.
282 changes: 272 additions & 10 deletions src/dcat-us/dataset-formatter.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,13 @@ it('dcatHelper: it does not allow customizations to overwrite critical fields',
identifier: 'SABOTAGE',
landingPage: 'SABOTAGE',
webService: 'SABOTAGE',
spatial: 'SABOTAGE',
};
const template = buildDatasetTemplate(customizations);
expect(template['@type']).not.toBe('SABOTAGE');
expect(template.contactPoint['@type']).not.toBe('SABOTAGE');
expect(template.identifier).not.toBe('SABOTAGE');
expect(template.landingPage).not.toBe('SABOTAGE');
expect(template.webService).not.toBe('SABOTAGE');
expect(template.spatial).not.toBe('SABOTAGE');
expect(template.title).toBe('{{metadata.metadata.name||item.title}}')
expect(template.contactPoint.fn).toBe('{{item.owner}}');
expect(template.contactPoint.hasEmail).toBe('mailto:dcat.support@dc.gov');
Expand All @@ -37,7 +35,7 @@ it('dcatHelper: it does not throw an error if there are no customizations', () =
});

it('dcatHelper: it does not throw an error customizations are null', () => {
const customizations = null;
const customizations = undefined;
const template = buildDatasetTemplate(customizations);
expect(template).toBeTruthy();
});
Expand Down Expand Up @@ -944,7 +942,7 @@ describe('formatDcatDataset', () => {
}
});

it('overwrites critical fields without values to an empty string', () => {
it('does not define spatial property when template does not specify it', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
Expand Down Expand Up @@ -977,11 +975,275 @@ describe('formatDcatDataset', () => {
},
};

try {
const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate()));
expect(formatted.spatial).toEqual('');
} catch {
fail('Should not throw!');
}
const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: undefined })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();
});

it('does not define spatial property adlib cannot find value to inject', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
modified: 1570747379000,
tags: ['Uno', 'Dos', 'Tres'],
name: 'DCAT_Test',
description: 'Some Description',
source: 'Test Source',
id: '00000000000000000000000000000000_0',
type: 'Feature Layer',
url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0',
layer: {
geometryType: 'esriGeometryPolygon',
},
server: {
spatialReference: {
wkid: 3310,
},
},
metadata: {
metadata: {
distInfo: {
distTranOps: {
onLineSrc: {
linkage: 'https://foobar.com',
},
},
},
},
},
};

const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{orgExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();
});

it('injects a spatial property equal to a hardcoded value', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
modified: 1570747379000,
tags: ['Uno', 'Dos', 'Tres'],
name: 'DCAT_Test',
description: 'Some Description',
source: 'Test Source',
id: '00000000000000000000000000000000_0',
type: 'Feature Layer',
url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0',
layer: {
geometryType: 'esriGeometryPolygon',
},
server: {
spatialReference: {
wkid: 3310,
},
},
metadata: {
metadata: {
distInfo: {
distTranOps: {
onLineSrc: {
linkage: 'https://foobar.com',
},
},
},
},
},
};

const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: 'hardcoded' })));
expect(formatted.spatial).toEqual('hardcoded');
});

it('injects a spatial property equal to a fallback hardcoded value', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
modified: 1570747379000,
tags: ['Uno', 'Dos', 'Tres'],
name: 'DCAT_Test',
description: 'Some Description',
source: 'Test Source',
id: '00000000000000000000000000000000_0',
type: 'Feature Layer',
url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0',
layer: {
geometryType: 'esriGeometryPolygon',
},
server: {
spatialReference: {
wkid: 3310,
},
},
metadata: {
metadata: {
distInfo: {
distTranOps: {
onLineSrc: {
linkage: 'https://foobar.com',
},
},
},
},
},
};

const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent || hardcoded}}' })));
expect(formatted.spatial).toEqual('hardcoded');
});

it('does not define spatial property if injected property is not a valid bbox', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
modified: 1570747379000,
tags: ['Uno', 'Dos', 'Tres'],
name: 'DCAT_Test',
description: 'Some Description',
source: 'Test Source',
id: '00000000000000000000000000000000_0',
type: 'Feature Layer',
url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0',
layer: {
geometryType: 'esriGeometryPolygon',
},
server: {
spatialReference: {
wkid: 3310,
},
},
metadata: {
metadata: {
distInfo: {
distTranOps: {
onLineSrc: {
linkage: 'https://foobar.com',
},
},
},
},
},
} as any;

dataset.itemExtent = [];
let formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[], []];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1], []];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1], [2]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1, 1], [2]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1], [2, 2]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [['1', 1], [2, 2]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1, '1'], [2, 2]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1, 1], ['2', 2]];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();

dataset.itemExtent = [[1, 1], [2, '2']];
formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.hasOwnProperty('spatial')).toBeFalsy();
});

it('injects spatial property from a datasets itemExtent', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
modified: 1570747379000,
tags: ['Uno', 'Dos', 'Tres'],
itemExtent: [[1,1], [2,2]],
name: 'DCAT_Test',
description: 'Some Description',
source: 'Test Source',
id: '00000000000000000000000000000000_0',
type: 'Feature Layer',
url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0',
layer: {
geometryType: 'esriGeometryPolygon',
},
server: {
spatialReference: {
wkid: 3310,
},
},
metadata: {
metadata: {
distInfo: {
distTranOps: {
onLineSrc: {
linkage: 'https://foobar.com',
},
},
},
},
},
} as any;

const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{itemExtent}}' })));
expect(formatted.spatial).toEqual('1.0000,1.0000,2.0000,2.0000');
});

it('injects spatial property from a datasets valid extent', () => {
const dataset = {
owner: 'fpgis.CALFIRE',
created: 1570747289000,
modified: 1570747379000,
tags: ['Uno', 'Dos', 'Tres'],
extent: {
type: 'envelope',
coordinates: [[1,1], [2,2]]
},
name: 'DCAT_Test',
description: 'Some Description',
source: 'Test Source',
id: '00000000000000000000000000000000_0',
type: 'Feature Layer',
url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0',
layer: {
geometryType: 'esriGeometryPolygon',
},
server: {
spatialReference: {
wkid: 3310,
},
},
metadata: {
metadata: {
distInfo: {
distTranOps: {
onLineSrc: {
linkage: 'https://foobar.com',
},
},
},
},
},
} as any;

const formatted = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ spatial: '{{extent}}' })));
expect(formatted.spatial).toEqual('1.0000,1.0000,2.0000,2.0000');
});
});
41 changes: 38 additions & 3 deletions src/dcat-us/dataset-formatter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { adlib, TransformsList } from 'adlib';
import { isPage } from '@esri/hub-sites';
import { baseDatasetTemplate } from './base-dataset-template';
import { _generateDistributions } from './_generate-distributions';
import { cloneObject, DatasetResource, datasetToContent, getContentSiteUrls, IModel } from '@esri/hub-common';
import { cloneObject, DatasetResource, datasetToContent, getContentSiteUrls, IModel, isBBox } from '@esri/hub-common';
import { IItem } from '@esri/arcgis-rest-portal';
import { nonEditableFieldPaths } from './noneditable-fields';

Expand Down Expand Up @@ -66,14 +66,17 @@ export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: st
downloadLink
});

if (_.has(hubDataset, 'extent.coordinates')) {
dcatDataset.spatial = hubDataset.extent.coordinates.join(',');
const spatial = computeSpatialProperty(datasetTemplate, dcatDataset);
if (spatial) {
dcatDataset.spatial = spatial;

// https://project-open-data.cio.gov/v1.1/schema/#theme
// allow theme to be overridden
if (_.isEmpty(datasetTemplate.theme)) {
dcatDataset.theme = ['geospatial'];
}
} else if (dcatDataset.spatial) {
delete dcatDataset.spatial;
}

return indent(JSON.stringify(dcatDataset, null, '\t'), 2);
Expand Down Expand Up @@ -116,4 +119,36 @@ function resetUninterpolatedPaths(dataset, fieldPaths) {
_.set(dataset, path, '');
}
});
}

/**
* Determines what to put in the spatial property based on template, raw dataset, and adlib-ed dataset
*/
function computeSpatialProperty(datasetTemplate, dcatDataset) {
// Either the template does not have spatial key set or somehow adlib does not inject, so don't set
if (!datasetTemplate.spatial || !dcatDataset.spatial) return undefined;

// Adlib returns the input for a templated value when it cannot resolve a non-falsey value
if (typeof dcatDataset.spatial === 'string' && dcatDataset.spatial.match(/{{.+}}/)?.length) return undefined;

// Get coordinates from valid GeoJSON bbox envelope or raw coordinates
let coordinates;
if (dcatDataset.spatial.type === 'envelope' && Array.isArray(dcatDataset.spatial.coordinates)) {
coordinates = dcatDataset.spatial.coordinates;
} else if (Array.isArray(dcatDataset.spatial)) {
coordinates = dcatDataset.spatial;
}

// Just return what adlib returned if coordinates cannot be obtained
if (!coordinates) return dcatDataset.spatial;

// If valid bbox coordinates return stringified
if (!isBBox(coordinates)) return undefined;
if (coordinates.length != 2 || coordinates[0].length != 2 || coordinates[1].length != 2) {
return undefined;
}
if (!_.isNumber(coordinates[0][0]) || !_.isNumber(coordinates[0][1]) || !_.isNumber(coordinates[1][0]) || !_.isNumber(coordinates[1][1])) {
return undefined;
}
return `${coordinates[0][0].toFixed(4)},${coordinates[0][1].toFixed(4)},${coordinates[1][0].toFixed(4)},${coordinates[1][1].toFixed(4)}`;
}
Loading

0 comments on commit 21189e3

Please sign in to comment.