Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update to keep track of superseded products to avoid false warnings #615

Merged
merged 6 commits into from
Mar 30, 2023
Merged

Update to keep track of superseded products to avoid false warnings #615

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0f1f73e
test to demonstrate problem
Mar 15, 2023
0f6cf3f
verify that references are not in skipped products
Mar 24, 2023
c0b697b
fix regression test and add it
Mar 27, 2023
4184251
fix sonatype
Mar 27, 2023
58191ae
add data we told git to ignore
Mar 27, 2023
ff9a811
sonatype fix
jordanpadams Mar 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/main/java/gov/nasa/pds/tools/validate/BundleManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ public static ArrayList<Target> buildBundleIgnoreList(URL url, String labelFileE
"Skipping " + target.getUrl() + " due to version not latest version"),
target.getUrl());
BundleManager.m_report.recordSkip(new URI(target.getUrl().toString()), p1);
SkippedItems.getInstance().add(target.getUrl());
} catch (Exception e) {
LOG.error(
"buildBundleIgnoreList:Cannot build ValidationProblem object or report skip file: {}",
Expand Down Expand Up @@ -738,6 +739,7 @@ public static ArrayList<Target> findOtherCollectionFiles(List<Target> targetList
+ " due to collection not latest or does not sharing the same logical_identifier as the bundle target"),
target.getUrl());
BundleManager.m_report.recordSkip(new URI(target.getUrl().toString()), p1);
SkippedItems.getInstance().add(target.getUrl());
} catch (Exception e) {
LOG.error(
"findOtherCollectionFiles:Cannot build ValidationProblem object or report skip file: {}",
Expand Down Expand Up @@ -804,6 +806,7 @@ public static void makeException(URL url, String location, String labelFileExten
+ target.getUrl() + " due to not being selected as the bundle target"),
target.getUrl());
BundleManager.m_report.recordSkip(new URI(target.getUrl().toString()), p1);
SkippedItems.getInstance().add(target.getUrl());
} catch (Exception e) {
LOG.error("makeException:Cannot build ValidationProblem object or report skip file: {}",
target.getUrl());
Expand Down
25 changes: 25 additions & 0 deletions src/main/java/gov/nasa/pds/tools/validate/SkippedItems.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package gov.nasa.pds.tools.validate;

import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

public class SkippedItems {
final private HashSet<URL> skipped = new HashSet<URL>();
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
private static SkippedItems singleton = null;
private SkippedItems() {
}
public static synchronized SkippedItems getInstance() {
jordanpadams marked this conversation as resolved.
Show resolved Hide resolved
if (singleton == null) singleton = new SkippedItems();
return singleton;
}
public synchronized void add (URL xmlfile) {
this.skipped.add(xmlfile);
}
public synchronized List<URL> copy() {
List<URL> duplicate = new ArrayList<URL>();
duplicate.addAll(this.skipped);
return duplicate;
}
}
47 changes: 43 additions & 4 deletions src/main/java/gov/nasa/pds/tools/validate/rule/pds4/FindUnreferencedFiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,16 @@
// $Id$
package gov.nasa.pds.tools.validate.rule.pds4;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import gov.nasa.pds.tools.validate.SkippedItems;
import gov.nasa.pds.tools.validate.rule.AbstractValidationRule;
import gov.nasa.pds.tools.validate.rule.GenericProblems;
import gov.nasa.pds.tools.validate.rule.ValidationTest;
Expand All @@ -26,13 +32,46 @@
*/
public class FindUnreferencedFiles extends AbstractValidationRule {
private static final Logger LOG = LoggerFactory.getLogger(FindUnreferencedIdentifiers.class);

private HashSet<String> buildReferencedFromSkipped() {
HashSet<String> referencedBySkipped = new HashSet<String>();
InputStream uis = null;
final String closeTag = "</file_name>", openTag = "<file_name>";
for (URL skipped : SkippedItems.getInstance().copy()) {
try {
int begin=0,end=-1;
uis = skipped.openStream();
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
String content = new String(uis.readAllBytes());
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
String path = this.extractPath(skipped);
begin = content.indexOf(openTag);

while (-1 < begin) {
end = content.indexOf(closeTag, begin);
referencedBySkipped.add("file:" + Paths.get(path, content.substring(begin+openTag.length(), end)).toString());
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
begin = content.indexOf(openTag, end + closeTag.length());
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
} finally {
if (uis != null) {
try {
uis.close();
} catch (IOException e) {}
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
uis = null;
}
}
}
return referencedBySkipped;
}
private String extractPath (URL url) {
Path fullpath = Paths.get(url.getPath());
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
return fullpath.getParent().toString();
}
@Override
public boolean isApplicable(String location) {
// This rule is applicable at the top level only.
return getContext().isRootTarget();
}

/**
* Iterate over unreferenced targets, reporting an error for each.
*/
Expand All @@ -46,11 +85,12 @@ public void findUnreferencedTargets() {
getRegistrar().getUnreferencedTargets().size());
// Only run the test if we are the root target, to avoid duplicate errors.
if (getContext().isRootTarget() && !getContext().getAllowUnlabeledFiles()) {
HashSet<String> referencedBySkipped = this.buildReferencedFromSkipped();
for (String location : getRegistrar().getUnreferencedTargets()) {
LOG.debug("findUnreferencedTargets:location: {}", location);
try {
// issue42 - to skip Product level validation
if (!getContext().getSkipProductValidation()) {
if (!getContext().getSkipProductValidation() && !referencedBySkipped.contains(location)) {
reportError(PDS4Problems.UNLABELED_FILE, new URL(location), -1, -1);
}
} catch (MalformedURLException e) {
Expand All @@ -60,5 +100,4 @@ public void findUnreferencedTargets() {
}
}
}

}
90 changes: 90 additions & 0 deletions src/test/resources/github597/bundle_mars2020_spice_v001.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-model href="http://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1500.sch"
schematypens="http://purl.oclc.org/dsdl/schematron"?>

<Product_Bundle xmlns="http://pds.nasa.gov/pds4/pds/v1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pds.nasa.gov/pds4/pds/v1 http://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1500.xsd">
<Identification_Area>
<logical_identifier>urn:nasa:pds:mars2020.spice</logical_identifier>
<version_id>1.0</version_id>
<title>Mars 2020 Perseverance Rover Mission SPICE Kernel Archive Bundle</title>
<information_model_version>1.5.0.0</information_model_version>
<product_class>Product_Bundle</product_class>
<Citation_Information>
<author_list>Costa Sitja M.; Semenov B. V.; Barnes M. J.</author_list>
<publication_year>2021</publication_year>
<keyword>Observation Geometry</keyword>
<description>This bundle contains Mars 2020 Perseverance Rover Mission SPICE kernels and related documentation.</description>
</Citation_Information>
</Identification_Area>
<Context_Area>
<Time_Coordinates>
<start_date_time>2020-07-30T12:51:34Z</start_date_time>
<stop_date_time>2021-05-21T15:47:08Z</stop_date_time>
</Time_Coordinates>
<Primary_Result_Summary>
<purpose>Observation Geometry</purpose>
<processing_level>Derived</processing_level>
</Primary_Result_Summary>
<Investigation_Area>
<name>Mars 2020 Perseverance Rover Mission</name>
<type>Mission</type>
<Internal_Reference>
<lid_reference>urn:nasa:pds:context:investigation:mission.mars2020</lid_reference>
<reference_type>bundle_to_investigation</reference_type>
</Internal_Reference>
</Investigation_Area>
<Observing_System>
<Observing_System_Component>
<name>Mars 2020 Perseverance Rover</name>
<type>Spacecraft</type>
<Internal_Reference>
<lid_reference>urn:nasa:pds:context:instrument_host:spacecraft.mars2020</lid_reference>
<reference_type>is_instrument_host</reference_type>
</Internal_Reference>
</Observing_System_Component>
</Observing_System>
<Target_Identification>
<name>MARS</name>
<type>Planet</type>
<Internal_Reference>
<lid_reference>urn:nasa:pds:context:target:planet.mars</lid_reference>
<reference_type>bundle_to_target</reference_type>
</Internal_Reference>
</Target_Identification>
</Context_Area>
<Reference_List>
<Internal_Reference>
<lid_reference>urn:nasa:pds:mars2020.spice:document:spiceds</lid_reference>
<reference_type>bundle_to_document</reference_type>
</Internal_Reference>
</Reference_List>
<Bundle>
<bundle_type>Archive</bundle_type>
<description>This bundle contains Mars 2020 Perseverance Rover Mission SPICE kernels and related documentation.</description>
</Bundle>
<File_Area_Text>
<File>
<file_name>readme.txt</file_name>
<creation_date_time>2021-08-20T11:52:01</creation_date_time>
<file_size unit="byte">1365</file_size>
<md5_checksum>03fb547e38c7f87d33d48123ae01d8b2</md5_checksum>
</File>
<Stream_Text>
<offset unit="byte">0</offset>
<parsing_standard_id>7-Bit ASCII Text</parsing_standard_id>
<record_delimiter>Carriage-Return Line-Feed</record_delimiter>
</Stream_Text>
</File_Area_Text>
<Bundle_Member_Entry>
<lidvid_reference>urn:nasa:pds:mars2020.spice:spice_kernels::1.0</lidvid_reference>
<member_status>Primary</member_status>
<reference_type>bundle_has_spice_kernel_collection</reference_type>
</Bundle_Member_Entry>
<Bundle_Member_Entry>
<lidvid_reference>urn:nasa:pds:mars2020.spice:document::1.0</lidvid_reference>
<member_status>Primary</member_status>
<reference_type>bundle_has_document_collection</reference_type>
</Bundle_Member_Entry>
</Product_Bundle>


88 changes: 88 additions & 0 deletions src/test/resources/github597/bundle_mars2020_spice_v002.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-model href="http://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1500.sch"
schematypens="http://purl.oclc.org/dsdl/schematron"?>

<Product_Bundle xmlns="http://pds.nasa.gov/pds4/pds/v1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pds.nasa.gov/pds4/pds/v1 http://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1500.xsd">
<Identification_Area>
<logical_identifier>urn:nasa:pds:mars2020.spice</logical_identifier>
<version_id>2.0</version_id>
<title>Mars 2020 Perseverance Rover Mission SPICE Kernel Archive Bundle</title>
<information_model_version>1.5.0.0</information_model_version>
<product_class>Product_Bundle</product_class>
<Citation_Information>
<author_list>Costa Sitja M.; Semenov B. V.; Barnes M. J.</author_list>
<publication_year>2021</publication_year>
<keyword>Observation Geometry</keyword>
<description>This bundle contains Mars 2020 Perseverance Rover Mission SPICE kernels and related documentation.</description>
</Citation_Information>
</Identification_Area>
<Context_Area>
<Time_Coordinates>
<start_date_time>2020-07-30T12:51:34Z</start_date_time>
<stop_date_time>2021-08-22T03:10:00Z</stop_date_time>
</Time_Coordinates>
<Primary_Result_Summary>
<purpose>Observation Geometry</purpose>
<processing_level>Derived</processing_level>
</Primary_Result_Summary>
<Investigation_Area>
<name>Mars 2020 Perseverance Rover Mission</name>
<type>Mission</type>
<Internal_Reference>
<lid_reference>urn:nasa:pds:context:investigation:mission.mars2020</lid_reference>
<reference_type>bundle_to_investigation</reference_type>
</Internal_Reference>
</Investigation_Area>
<Observing_System>
<Observing_System_Component>
<name>Mars 2020 Perseverance Rover</name>
<type>Spacecraft</type>
<Internal_Reference>
<lid_reference>urn:nasa:pds:context:instrument_host:spacecraft.mars2020</lid_reference>
<reference_type>is_instrument_host</reference_type>
</Internal_Reference>
</Observing_System_Component>
</Observing_System>
<Target_Identification>
<name>MARS</name>
<type>Planet</type>
<Internal_Reference>
<lid_reference>urn:nasa:pds:context:target:planet.mars</lid_reference>
<reference_type>bundle_to_target</reference_type>
</Internal_Reference>
</Target_Identification>
</Context_Area>
<Reference_List>
<Internal_Reference>
<lid_reference>urn:nasa:pds:mars2020.spice:document:spiceds</lid_reference>
<reference_type>bundle_to_document</reference_type>
</Internal_Reference>
</Reference_List>
<Bundle>
<bundle_type>Archive</bundle_type>
<description>This bundle contains Mars 2020 Perseverance Rover Mission SPICE kernels and related documentation.</description>
</Bundle>
<File_Area_Text>
<File>
<file_name>readme.txt</file_name>
<creation_date_time>2021-11-19T07:20:24</creation_date_time>
<file_size unit="byte">1365</file_size>
<md5_checksum>03fb547e38c7f87d33d48123ae01d8b2</md5_checksum>
</File>
<Stream_Text>
<offset unit="byte">0</offset>
<parsing_standard_id>7-Bit ASCII Text</parsing_standard_id>
<record_delimiter>Carriage-Return Line-Feed</record_delimiter>
</Stream_Text>
</File_Area_Text>
<Bundle_Member_Entry>
<lidvid_reference>urn:nasa:pds:mars2020.spice:spice_kernels::2.0</lidvid_reference>
<member_status>Primary</member_status>
<reference_type>bundle_has_spice_kernel_collection</reference_type>
</Bundle_Member_Entry>
<Bundle_Member_Entry>
<lidvid_reference>urn:nasa:pds:mars2020.spice:document::1.0</lidvid_reference>
<member_status>Secondary</member_status>
<reference_type>bundle_has_document_collection</reference_type>
</Bundle_Member_Entry>
</Product_Bundle>
Loading