From 36fc837a57013740e56ba63c123ea8b1ff0d4a4b Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 24 Apr 2023 12:36:45 -0700 Subject: [PATCH 1/5] add unit test --- src/test/resources/features/validate.feature | 3 + .../resources/github628/mp2_flat_20061109.xml | 93 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/test/resources/github628/mp2_flat_20061109.xml diff --git a/src/test/resources/features/validate.feature b/src/test/resources/features/validate.feature index fdc7ed646..40a5ba81a 100644 --- a/src/test/resources/features/validate.feature +++ b/src/test/resources/features/validate.feature @@ -9,6 +9,9 @@ Scenario Outline: Execute validate command for tests below. Examples: | testName | testDir | messageCount | messageText | problemEnum | resourceDir | reportDir | commandArgs | refOutputValue | +# Validate#628 +|"NASA-PDS/validate#616 Warning Version Mismatch" | "github628" | 1 | "1 warnings expected" | "totalWarnings" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github628.json -s json --skip-content-validation -t {resourceDir}/github628/mp2_flat_20061109.xml" | "report_github628.json" | + # Validate#616 |"NASA-PDS/validate#616 Success Multiple Tables One File" | "github616" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github616.json -s json --skip-context-validation -t {resourceDir}/github616/mre_cal_sc_ttcp_delay_schulte_01s_2021069.xml" | "report_github616.json" | diff --git a/src/test/resources/github628/mp2_flat_20061109.xml b/src/test/resources/github628/mp2_flat_20061109.xml new file mode 100644 index 000000000..eefa04718 --- /dev/null +++ b/src/test/resources/github628/mp2_flat_20061109.xml @@ -0,0 +1,93 @@ + + + + + + + urn:nasa:pds:nh_mvic:calibration_files:mp2_flat + 1.0 + Flat Field for the New Horizons MVIC Instrument Panchromatic CCD #2 in Time Delay Integration (TDI) Mode, 2006-11-09 version + 1.18.0.0 + Product_Ancillary + + + 2022-11 + 1.0 + + A. Raugh: Migrated from PDS3. Note that there are later version + of this file, used to produce MVIC collections later + in the mission lifetime. Labels for calibrated MVIC + data should reference the specific version used in + producing that product. + + + + + + + + Calibration + Derived + Flat Field + + Visible + Imaging + + + + + New Horizons Kuiper Belt Extended Mission 1 + Mission + + urn:nasa:pds:context:investigation:mission.new_horizons_kem1 + ancillary_to_investigation + + + + + + Multispectral Visible Imaging Camera (MVIC) for New Horizons + Instrument + + urn:nasa:pds:context:instrument:nh.mvic + is_instrument + + + + + + + + mp2_flat_20061109.fit + + This file contains the flat field used in calibrating the MVIC + panchromatic CCD #2 image data. + + The digits in the file name indicate the approximate date (YYYYMMDD) + of the delivery of the file for use in pipelines. + + +
+ 0 + 2880 + FITS 3.0 +
+ + FlatField + 2880 + 1 + Last Index Fastest + + IEEE754MSBDouble + + + Sample + 5024 + 1 + + + + + From e5bb32050ff641ac487f692ce5b6a4a270c28a39 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 24 Apr 2023 12:42:53 -0700 Subject: [PATCH 2/5] add unit test --- src/test/resources/features/validate.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/features/validate.feature b/src/test/resources/features/validate.feature index 40a5ba81a..1f49d3146 100644 --- a/src/test/resources/features/validate.feature +++ b/src/test/resources/features/validate.feature @@ -10,7 +10,7 @@ Scenario Outline: Execute validate command for tests below. | testName | testDir | messageCount | messageText | problemEnum | resourceDir | reportDir | commandArgs | refOutputValue | # Validate#628 -|"NASA-PDS/validate#616 Warning Version Mismatch" | "github628" | 1 | "1 warnings expected" | "totalWarnings" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github628.json -s json --skip-content-validation -t {resourceDir}/github628/mp2_flat_20061109.xml" | "report_github628.json" | +|"NASA-PDS/validate#628 Warning Version Mismatch" | "github628" | 1 | "1 warnings expected" | "totalWarnings" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github628.json -s json --skip-content-validation -t {resourceDir}/github628/mp2_flat_20061109.xml" | "report_github628.json" | # Validate#616 |"NASA-PDS/validate#616 Success Multiple Tables One File" | "github616" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github616.json -s json --skip-context-validation -t {resourceDir}/github616/mre_cal_sc_ttcp_delay_schulte_01s_2021069.xml" | "report_github616.json" | From 7332bec3d7be3469444b0452e86ef94f16781823 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 27 Apr 2023 09:00:34 -0700 Subject: [PATCH 3/5] add check for schema/schematron versioning --- .../nasa/pds/tools/label/LabelValidator.java | 84 ++++++++++++++++--- 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java b/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java index 7514a39de..61a80f4e1 100644 --- a/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java +++ b/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java @@ -19,10 +19,14 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.xml.XMLConstants; @@ -81,6 +85,7 @@ import gov.nasa.pds.tools.validate.ValidationProblem; import gov.nasa.pds.validate.constants.Constants; import net.sf.saxon.om.DocumentInfo; +import net.sf.saxon.trans.XPathException; /** * This class is responsible for providing utility functions for validating PDS XML Labels. @@ -120,8 +125,6 @@ public class LabelValidator { private SchematronTransformer schematronTransformer; private XPathFactory xPathFactory; - private String labelExtension; - private long filesProcessed = 0; private double totalTimeElapsed = 0.0; @@ -179,8 +182,6 @@ public LabelValidator() throws ParserConfigurationException, TransformerConfigur // The parser doesn't validate - we use a Validator instead. saxParserFactory.setValidating(false); - setLabelExtension(Constants.DEFAULT_LABEL_EXTENSION); - // Don't add xml:base attributes to xi:include content, or it messes up // PDS4 validation. try { @@ -328,7 +329,6 @@ public synchronized void validate(ProblemHandler handler, URL url) throws SAXExc public synchronized void validate(ProblemHandler handler, URL url, String labelExtension) throws SAXException, IOException, ParserConfigurationException, TransformerException, MissingLabelSchemaException { - this.setLabelExtension(labelExtension); this.parseAndValidate(handler, url); } @@ -383,6 +383,73 @@ private Boolean determineSchematronValidationFlag(URL url) { return (validateAgainstSchematronFlag); } + private void checkSchemaSchematronVersions (ProblemHandler handler, URL url) { + try { + List specifiedSchema=null, specifiedSchematron=null; + XMLExtractor sourceXML = new XMLExtractor(url); + String xmlns = sourceXML.getSchemaLocation(); + specifiedSchematron = sourceXML.getXmlModels(); + if (xmlns != null && 0 < xmlns.strip().length()) specifiedSchema = Arrays.asList(xmlns.split("\\s")); + if (specifiedSchema == null || specifiedSchema.size() == 0) { + handler.addProblem(new ValidationProblem( + new ProblemDefinition(ExceptionType.WARNING, ProblemType.SCHEMA_WARNING, "Cannot check versioning because no Schema given."), + url)); + } else if (specifiedSchematron == null || specifiedSchematron.size() == 0) { + handler.addProblem(new ValidationProblem( + new ProblemDefinition(ExceptionType.WARNING, ProblemType.SCHEMATRON_WARNING, "Cannot check versioning because no Schematron given."), + url)); + } else { + Set schemas = new HashSet(); + Set schematrons = new HashSet(); + for (String schema : specifiedSchema) { + try { + URL schemaURL = new URL(schema); + String schemaName = Paths.get(schemaURL.getPath()).getFileName().toString().toLowerCase(); + if (schemaName.endsWith(".xsd")) { + schemaName = schemaName.substring(0, schemaName.length()-4); + schemas.add(schemaName); + } + } catch (MalformedURLException e) { + // Ignore error since XMLExtractor() was able to read it this should be alright + } + } + for (String model : specifiedSchematron) { + for (String part : model.split("\\s")) { + if (part.toLowerCase().startsWith("href")) { + String modelHREF = part.substring(part.indexOf('"')+1, part.lastIndexOf('"')); + try { + URL schematronURL = new URL(modelHREF); + String schematronName = Paths.get(schematronURL.getPath()).getFileName().toString().toLowerCase(); + if (schematronName.endsWith(".sch")) { + schematronName = schematronName.substring(0, schematronName.length()-4); + schematrons.add(schematronName); + } + } catch (MalformedURLException e) { + // Ignore error since XMLExtractor() was able to it this should be alright + } + } + } + } + Set uniqueSchema = new HashSet(schemas); + Set uniqueSchematron = new HashSet(schematrons); + uniqueSchema.removeAll(schematrons); + uniqueSchematron.removeAll(schemas); + if (0 < uniqueSchema.size() + uniqueSchematron.size()) { + handler.addProblem(new ValidationProblem( + new ProblemDefinition(ExceptionType.WARNING, ProblemType.SCHEMA_WARNING, + "The schema version(s) " + uniqueSchema.toString() + + " does/do not match the schematron version(s) " + uniqueSchematron.toString() + "."), + url)); + } + } + } + catch (XPathException | XPathExpressionException e) { + handler.addProblem(new ValidationProblem( + new ProblemDefinition(ExceptionType.WARNING, ProblemType.MISSING_REQUIRED_RESOURCE, "Cannot check versioning because XML could not be parsed."), + url)); + } + } + /** * Parses and validates a label against the schema and Schematron files, and returns the parsed * XML. @@ -415,6 +482,7 @@ public synchronized Document parseAndValidate(ProblemHandler handler, URL url) // Are we perfoming schema validation? if (performsSchemaValidation()) { createParserIfNeeded(handler); + checkSchemaSchematronVersions(handler, url); // Do we need this to clear the cache? @@ -970,10 +1038,6 @@ public void setCachedLSResourceResolver(CachedLSResourceResolver resolver) { this.cachedLSResolver = resolver; } - public void setLabelExtension(String extension) { - this.labelExtension = extension; - } - public Pattern getBundleLabelPattern() { return bundleLabelPattern; } @@ -1007,7 +1071,6 @@ public static void main(String[] args) throws Exception { */ private static class DOMLocator implements Locator { - private URL url; private int lineNumber; private int columnNumber; private String systemId; @@ -1018,7 +1081,6 @@ private static class DOMLocator implements Locator { * @param url the URL of the source document */ public DOMLocator(URL url) { - this.url = url; this.systemId = url.toString(); } From 80406c9f74e4795b96f5cada26d5e74d90aaf981 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 27 Apr 2023 09:13:32 -0700 Subject: [PATCH 4/5] introduced new error to old check --- src/test/resources/features/validate.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/features/validate.feature b/src/test/resources/features/validate.feature index 1f49d3146..ac34d31bb 100644 --- a/src/test/resources/features/validate.feature +++ b/src/test/resources/features/validate.feature @@ -93,7 +93,7 @@ Scenario Outline: Execute validate command for tests below. # So, the 1st error is the file name contains spaces from FileAndDirectoryNamingRule and the 2nd error comes from function validateFileStandardConformity(). |"NASA-PDS/validate#153 1" | "github153" | 2 | "2 error messages expected." | "totalErrors" | "src/test/resources" | "target/test" | "--skip-context-validation -R pds4.label -r {reportDir}/report_github153_invalid.json -s json -t {resourceDir}/github153/iue_asteroid_spectra/document/3juno_lwr01896_ines_fits_headers.pdfa%.xml" | "report_github153_invalid.json" | |"NASA-PDS/validate#153 2" | "github153" | 0 | "No error messages expected." | "totalErrors" | "src/test/resources" | "target/test" | "--skip-context-validation -R pds4.label -r {reportDir}/report_github153_valid.json -s json -t {resourceDir}/github153/iue_asteroid_spectra/document/collection_iue_asteroid_spectra_document.xml" | "report_github153_valid.json" | - |"NASA-PDS/validate#17 1" | "github17" | 2 | "2 warning expected." | "totalWarnings" | "src/test/resources" | "target/test" | "--skip-context-validation -R pds4.label -r {reportDir}/report_github17_invalid.json -s json -t {resourceDir}/github17/delimited_table_bad.xml" | "report_github17_invalid.json" | + |"NASA-PDS/validate#17 1" | "github17" | 3 | "3 warning expected." | "totalWarnings" | "src/test/resources" | "target/test" | "--skip-context-validation -R pds4.label -r {reportDir}/report_github17_invalid.json -s json -t {resourceDir}/github17/delimited_table_bad.xml" | "report_github17_invalid.json" | |"NASA-PDS/validate#17 2" | "github17" | 0 | "No error messages expected." | "totalErrors" | "src/test/resources" | "target/test" | "--skip-context-validation -R pds4.label -r {reportDir}/report_github17_valid.json -s json -t {resourceDir}/github17/delimited_table_good.xml" | "report_github17_valid.json" | |"NASA-PDS/validate#230 1" | "github230" | 2 | "2 errors expected for MISSING_VERSION." | "MISSING_VERSION" | "src/test/resources" | "target/test" | "--skip-content-validation -R pds4.bundle -r {reportDir}/report_github230_invalid.json -s json -t {resourceDir}/github230/invalid/cocirs_c2h4abund/bundle_cocirs_c2h4abund.xml" | "report_github230_invalid.json" | From 1642a00146bf7bb9614463beda795ab2a5760fac Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Fri, 5 May 2023 15:16:17 -0700 Subject: [PATCH 5/5] Update log output to be upper case --- .../nasa/pds/tools/label/LabelValidator.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java b/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java index 61a80f4e1..c6c08e8e2 100644 --- a/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java +++ b/src/main/java/gov/nasa/pds/tools/label/LabelValidator.java @@ -383,20 +383,20 @@ private Boolean determineSchematronValidationFlag(URL url) { return (validateAgainstSchematronFlag); } - private void checkSchemaSchematronVersions (ProblemHandler handler, URL url) { + private void checkSchemaSchematronVersions(ProblemHandler handler, URL url) { try { - List specifiedSchema=null, specifiedSchematron=null; + List specifiedSchema = null, specifiedSchematron = null; XMLExtractor sourceXML = new XMLExtractor(url); String xmlns = sourceXML.getSchemaLocation(); specifiedSchematron = sourceXML.getXmlModels(); - if (xmlns != null && 0 < xmlns.strip().length()) specifiedSchema = Arrays.asList(xmlns.split("\\s")); + if (xmlns != null && 0 < xmlns.strip().length()) + specifiedSchema = Arrays.asList(xmlns.split("\\s")); if (specifiedSchema == null || specifiedSchema.size() == 0) { - handler.addProblem(new ValidationProblem( - new ProblemDefinition(ExceptionType.WARNING, ProblemType.SCHEMA_WARNING, "Cannot check versioning because no Schema given."), - url)); + handler.addProblem(new ValidationProblem(new ProblemDefinition(ExceptionType.WARNING, + ProblemType.SCHEMA_WARNING, "Cannot check versioning because no Schema given."), url)); } else if (specifiedSchematron == null || specifiedSchematron.size() == 0) { - handler.addProblem(new ValidationProblem( - new ProblemDefinition(ExceptionType.WARNING, ProblemType.SCHEMATRON_WARNING, "Cannot check versioning because no Schematron given."), + handler.addProblem(new ValidationProblem(new ProblemDefinition(ExceptionType.WARNING, + ProblemType.SCHEMATRON_WARNING, "Cannot check versioning because no Schematron given."), url)); } else { Set schemas = new HashSet(); @@ -404,9 +404,10 @@ private void checkSchemaSchematronVersions (ProblemHandler handler, URL url) { for (String schema : specifiedSchema) { try { URL schemaURL = new URL(schema); - String schemaName = Paths.get(schemaURL.getPath()).getFileName().toString().toLowerCase(); + String schemaName = + Paths.get(schemaURL.getPath()).getFileName().toString().toLowerCase(); if (schemaName.endsWith(".xsd")) { - schemaName = schemaName.substring(0, schemaName.length()-4); + schemaName = schemaName.substring(0, schemaName.length() - 4); schemas.add(schemaName); } } catch (MalformedURLException e) { @@ -416,12 +417,13 @@ private void checkSchemaSchematronVersions (ProblemHandler handler, URL url) { for (String model : specifiedSchematron) { for (String part : model.split("\\s")) { if (part.toLowerCase().startsWith("href")) { - String modelHREF = part.substring(part.indexOf('"')+1, part.lastIndexOf('"')); + String modelHREF = part.substring(part.indexOf('"') + 1, part.lastIndexOf('"')); try { URL schematronURL = new URL(modelHREF); - String schematronName = Paths.get(schematronURL.getPath()).getFileName().toString().toLowerCase(); + String schematronName = + Paths.get(schematronURL.getPath()).getFileName().toString().toLowerCase(); if (schematronName.endsWith(".sch")) { - schematronName = schematronName.substring(0, schematronName.length()-4); + schematronName = schematronName.substring(0, schematronName.length() - 4); schematrons.add(schematronName); } } catch (MalformedURLException e) { @@ -437,15 +439,16 @@ private void checkSchemaSchematronVersions (ProblemHandler handler, URL url) { if (0 < uniqueSchema.size() + uniqueSchematron.size()) { handler.addProblem(new ValidationProblem( new ProblemDefinition(ExceptionType.WARNING, ProblemType.SCHEMA_WARNING, - "The schema version(s) " + uniqueSchema.toString() + - " does/do not match the schematron version(s) " + uniqueSchematron.toString() + "."), + "The schema version(s) " + uniqueSchema.toString().toUpperCase() + + " does/do not match the schematron version(s) " + + uniqueSchematron.toString().toUpperCase() + "."), url)); } } - } - catch (XPathException | XPathExpressionException e) { + } catch (XPathException | XPathExpressionException e) { handler.addProblem(new ValidationProblem( - new ProblemDefinition(ExceptionType.WARNING, ProblemType.MISSING_REQUIRED_RESOURCE, "Cannot check versioning because XML could not be parsed."), + new ProblemDefinition(ExceptionType.WARNING, ProblemType.MISSING_REQUIRED_RESOURCE, + "Cannot check versioning because XML could not be parsed."), url)); } }