Skip to content

Commit

Permalink
Disallow ingestion package loading outside of footprint
Browse files Browse the repository at this point in the history
  • Loading branch information
weisenje authored and GitHub Enterprise committed Apr 5, 2023
1 parent 3b364f5 commit 26ceb4d
Show file tree
Hide file tree
Showing 8 changed files with 104 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public class LoadDataConfig extends YamlConfig {
private LinkedList<IngestionStep> steps = new LinkedList<IngestionStep>();
private String modelgraph; // schema supports multiple model graphs, but functionality not needed
private LinkedList<String> datagraphs;
private LinkedList<String> allowedGraphs; // if not null, only allow loads to these graphs

/**
* Constructor
Expand Down Expand Up @@ -114,6 +115,9 @@ public void addDatagraph(String datagraph) {
}
this.datagraphs.add(datagraph);
}
public void setAllowedGraphs(LinkedList<String> allowedGraphs) {
this.allowedGraphs = allowedGraphs;
}


/**
Expand All @@ -140,6 +144,7 @@ public void load(String modelGraph, LinkedList<String> dataGraphs, String server
SparqlConnection conn = new SparqlConnection();
conn.addModelInterface(serverType, server, modelGraph);
for(String dg : dataGraphs) {
if(allowedGraphs != null && !allowedGraphs.contains(dg)) { throw new Exception("Cannot load data from " + getFileName() + " in " + getBaseDir() + ": " + dg + " is not in list of allowed graphs: " + allowedGraphs); }
conn.addDataInterface(serverType, server, dg);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public LoadOwlConfig(File yamlFile, String defaultModelGraph) throws Exception {

private String modelgraph; // schema supports multiple model graphs, but functionality not needed
private LinkedList<String> files = new LinkedList<String>();
private LinkedList<String> allowedGraphs; // if not null, only allow loads to these graphs

/**
* Get methods
Expand All @@ -73,6 +74,9 @@ public void setModelgraph(String modelgraph) {
public void addFile(String file) {
this.files.add(file);
}
public void setAllowedGraphs(LinkedList<String> allowedGraphs) {
this.allowedGraphs = allowedGraphs;
}

/**
* Ingest the OWL files
Expand All @@ -86,6 +90,7 @@ public void load(String modelGraph, String server, String serverType, SparqlQuer
// use modelGraph from method parameter if present. Else use from config YAML if present. Else use default.
modelGraph = (modelGraph != null) ? modelGraph : (this.getModelgraph() != null ? this.getModelgraph() : this.defaultModelGraph );
if(modelGraph == null) { throw new Exception ("No model graph found"); }
if(allowedGraphs != null && !allowedGraphs.contains(modelGraph)) { throw new Exception("Cannot load from " + getFileName() + " in " + getBaseDir() + ": " + modelGraph + " is not in list of allowed graphs: " + allowedGraphs); }

// upload each OWL file to model graph
for(String fileStr : this.getFiles()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,14 @@ public void load(String server, String serverTypeString, boolean clear, boolean
// load via an owl ingestion YAML
File stepFile = new File(baseDir, (String)step.getValue());
LoadOwlConfig config = new LoadOwlConfig(stepFile, this.defaultModelGraph);
config.setAllowedGraphs(getGraphsFootprint());
config.load(null, server, serverTypeString, queryClient, logger);

}else if(type == StepType.DATA) {
// load content using CSV ingestion YAML
File stepFile = new File(baseDir, (String)step.getValue());
LoadDataConfig config = new LoadDataConfig(stepFile, this.defaultModelGraph, this.defaultDataGraph);
config.setAllowedGraphs(getGraphsFootprint());
config.load(null, null, server, serverTypeString, false, ingestClient, ngeClient, queryClient, logger);

}else if(type == StepType.NODEGROUPS) {
Expand All @@ -263,6 +265,9 @@ public void load(String server, String serverTypeString, boolean clear, boolean
// load content using sub-manifest
File stepFile = new File(baseDir, (String)step.getValue());
ManifestConfig subManifest = new ManifestConfig(stepFile, defaultModelGraph, defaultDataGraph);
if(!getGraphsFootprint().containsAll(subManifest.getGraphsFootprint())){ // require sub-manifest footprint to be subset of parent footprint
throw new Exception("Sub-manifest footprint " + subManifest.getGraphsFootprint() + " is not a subset of parent footprint " + getGraphsFootprint());
}
subManifest.load(server, serverTypeString, false, false, ingestClient, ngeClient, ngStoreClient, queryClient, logger);

}else if(type == StepType.COPYGRAPH) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/
public abstract class YamlConfig {

protected String fileName; // the YAML file name
protected String baseDir; // the directory containing the YAML file
protected String defaultModelGraph; // load to this model graph if not otherwise specified
protected String defaultDataGraph; // load to this data graph if not otherwise specified
Expand All @@ -23,9 +24,10 @@ public abstract class YamlConfig {
* Constructor
*/
public YamlConfig(File yamlFile, File schemaFile, String defaultModelGraph, String defaultDataGraph) throws Exception {
setBaseDir(yamlFile.getParent());
setDefaultModelGraph(defaultModelGraph);
setDefaultDataGraph(defaultDataGraph);
this.fileName = yamlFile.getName();
this.baseDir = yamlFile.getParent();
this.defaultModelGraph = defaultModelGraph;
this.defaultDataGraph = defaultDataGraph;

// validate manifest YAML against schema
String yamlStr = Utility.getStringFromFilePath(yamlFile.getAbsolutePath());
Expand All @@ -38,6 +40,9 @@ public YamlConfig(File yamlFile, File schemaFile, String defaultModelGraph, Stri
/**
* Get methods
*/
public String getFileName() {
return fileName;
}
public String getBaseDir() {
return baseDir;
}
Expand All @@ -48,19 +53,6 @@ public String getDefaultDataGraph() {
return defaultDataGraph;
}

/**
* Set methods
*/
protected void setBaseDir(String baseDir) {
this.baseDir = baseDir;
}
protected void setDefaultModelGraph(String defaultModelGraph) {
this.defaultModelGraph = defaultModelGraph;
}
protected void setDefaultDataGraph(String defaultDataGraph) {
this.defaultDataGraph = defaultDataGraph;
}

/**
* Utility method for a node that may be a string or a string array.
* Returns the string or the first array element, and errors if the array has multiple elements.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,25 @@ public void testLoad() throws Exception{
assertEquals(dataSei.getNumTriples(), 0);
assertEquals(dataFallbackSei.getNumTriples(), NUM_EXPECTED_TRIPLES);

// test specifying allowed graphs, where graphs to be loaded are present in the allowed graphs
LinkedList<String> allowedGraphs = new LinkedList<String>(Arrays.asList(modelSei.getGraph(), dataSei.getGraph()));
loadOwlConfig.setAllowedGraphs(allowedGraphs);
loadDataConfig.setAllowedGraphs(allowedGraphs);
loadOwlConfig.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); // loads OWL
loadDataConfig.load(modelSei.getGraph(), new LinkedList<String>(Arrays.asList(dataSei.getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);

// test specifying allowed graphs, where graphs to be loaded are NOT present in the allowed graphs
try {
allowedGraphs = new LinkedList<String>(Arrays.asList(modelSei.getGraph(), dataSei.getGraph() + "-123"));
loadOwlConfig.setAllowedGraphs(allowedGraphs);
loadDataConfig.setAllowedGraphs(allowedGraphs);
loadOwlConfig.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null); // loads OWL
loadDataConfig.load(modelSei.getGraph(), new LinkedList<String>(Arrays.asList(dataSei.getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
fail();
}catch(Exception e) {
assertTrue(e.getMessage().contains(dataSei.getGraph() + " is not in list of allowed graphs: [" + modelSei.getGraph() + ", " + dataSei.getGraph() + "-123]"));
}

}finally {
clearGraphs();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@
import static org.junit.Assert.fail;

import java.io.File;
import java.io.PrintWriter;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.LinkedList;

import org.junit.Test;

Expand Down Expand Up @@ -112,6 +113,19 @@ public void testLoad() throws Exception{
assertEquals(modelSei.getNumTriples(), 0);
assertEquals(modelFallbackSei.getNumTriples(), NUM_EXPECTED_TRIPLES);

// test specifying allowed graphs, where graphs to be loaded are present in the allowed graphs
config.setAllowedGraphs(new LinkedList<String>(Arrays.asList(modelSei.getGraph())));
config.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);

// test specifying allowed graphs, where graphs to be loaded are NOT present in the allowed graphs
config.setAllowedGraphs(new LinkedList<String>(Arrays.asList(modelSei.getGraph() + "-123")));
try {
config.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
fail();
}catch(Exception e) {
assertTrue(e.getMessage().contains(modelSei.getGraph() + " is not in list of allowed graphs: [" + modelSei.getGraph() + "-123]"));
}

}catch(Exception e) {
throw e;
}finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
package com.ge.research.semtk.load.config.test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;

import java.io.File;
import java.io.PrintWriter;

import org.apache.commons.io.FileUtils;
import org.apache.commons.math3.util.Pair;
Expand Down Expand Up @@ -125,6 +125,52 @@ public void testLoadManifest_UsesDefaultGraph() throws Exception{
}
}

/**
* Confirm error if sub-manifest footprint is not a subset of parent manifest footprint
*/
@Test
public void testLoadManifest_ErrorIfSubmanifestFootprintNotSubset() throws Exception{
File tempDir = null;
try {
tempDir = TestGraph.unzipAndUniquifyJunitGraphs(this, "/config/IngestionPackage-SubManifestFootprintNotSubset.zip");
ManifestConfig manifest = new ManifestConfig(ManifestConfig.getTopLevelManifestFile(tempDir), modelFallbackSei.getGraph(), dataFallbackSei.getGraph());
manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
}catch(Exception e) {
assertTrue(e.getMessage().contains("is not a subset of parent footprint"));
}finally{
if(tempDir != null) { FileUtils.deleteDirectory(tempDir); }
reset();
}
}

/**
* Confirm error if try to load to a graph that is not in the footprint
*/
@Test
public void testLoadManifest_ErrorLoadToGraphNotInFootprint() throws Exception{
// Note: no copy to default graph, so don't have to limit to Fuseki only

// using generic fallbacks don't match the footprint
SparqlEndpointInterface modelFallbackSei_notUnique = TestGraph.getSei("http://junit/rack001/model");
SparqlEndpointInterface dataFallbackSei_notUnique = TestGraph.getSei("http://junit/rack001/data");

File tempDir = null;
try {
// get manifest from ingestion package, perform load
tempDir = TestGraph.unzipAndUniquifyJunitGraphs(this, "/config/IngestionPackage.zip");
ManifestConfig manifest = new ManifestConfig(ManifestConfig.getTopLevelManifestFile(tempDir), modelFallbackSei_notUnique.getGraph(), dataFallbackSei_notUnique.getGraph());

// tries loading to fallback, which is not in the footprint
manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);

}catch(Exception e) {
assertTrue(e.getMessage().contains(modelFallbackSei_notUnique.getGraph() + " is not in list of allowed graphs: ["));
}finally{
if(tempDir != null) { FileUtils.deleteDirectory(tempDir); }
reset();
}
}

// clear graphs and nodegroup store
private void reset() throws Exception {
IntegrationTestUtility.clearGraph(modelFallbackSei);
Expand Down
Binary file not shown.

0 comments on commit 26ceb4d

Please sign in to comment.