Merge branch 'master' of github.build.ge.com:semtk/semtk-opensource

connectionUtils/src/main/java/com/ge/research/semtk/utility/Logger.java

@@ -59,6 +59,7 @@ public static String getMessage(String line) {
 
 	// log the message
 	private void log(String s, Levels level) {
+		s = s.replaceAll("\n", " ");  // remove newlines, to keep the message intact and associated with its level
 		printWriter.println(level.name() + ": " + s);
 		printWriter.flush();
 	}

sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadDataConfig.java

@@ -40,6 +40,7 @@ public class LoadDataConfig extends YamlConfig {
 	private LinkedList<IngestionStep> steps = new LinkedList<IngestionStep>();
 	private String modelgraph;   			// schema supports multiple model graphs, but functionality not needed
 	private LinkedList<String> datagraphs;
+	private LinkedList<String> allowedGraphs;	// if not null, only allow loads to these graphs
 
 	/**
 	 * Constructor
@@ -114,6 +115,9 @@ public void addDatagraph(String datagraph) {
 		}
 		this.datagraphs.add(datagraph);
 	}
+	public void setAllowedGraphs(LinkedList<String> allowedGraphs) {
+		this.allowedGraphs = allowedGraphs;
+	}
 
 
 	/**
@@ -140,6 +144,7 @@ public void load(String modelGraph, LinkedList<String> dataGraphs, String server
 			SparqlConnection conn = new SparqlConnection();
 			conn.addModelInterface(serverType, server, modelGraph);
 			for(String dg : dataGraphs) {
+				if(allowedGraphs != null && !allowedGraphs.contains(dg)) { throw new Exception("Cannot load data from " + getFileName() + " in " + getBaseDir()  + ": " + dg + " is not in list of allowed graphs: " + allowedGraphs); }
 				conn.addDataInterface(serverType, server, dg);
 			}
 

sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/LoadOwlConfig.java

@@ -53,6 +53,7 @@ public LoadOwlConfig(File yamlFile, String defaultModelGraph) throws Exception {
 
 	private String modelgraph;   // schema supports multiple model graphs, but functionality not needed
 	private LinkedList<String> files = new LinkedList<String>();
+	private LinkedList<String> allowedGraphs;	// if not null, only allow loads to these graphs
 
 	/**
 	 * Get methods
@@ -73,6 +74,9 @@ public void setModelgraph(String modelgraph) {
 	public void addFile(String file) {
 		this.files.add(file);
 	}
+	public void setAllowedGraphs(LinkedList<String> allowedGraphs) {
+		this.allowedGraphs = allowedGraphs;
+	}
 
 	/**
 	 * Ingest the OWL files
@@ -86,6 +90,7 @@ public void load(String modelGraph, String server, String serverType, SparqlQuer
 			// use modelGraph from method parameter if present.  Else use from config YAML if present.  Else use default.
 			modelGraph = (modelGraph != null) ? modelGraph : (this.getModelgraph() != null ? this.getModelgraph() : this.defaultModelGraph );
 			if(modelGraph == null) { throw new Exception ("No model graph found"); }
+			if(allowedGraphs != null && !allowedGraphs.contains(modelGraph)) { throw new Exception("Cannot load from " + getFileName() + " in " + getBaseDir() + ": " + modelGraph + " is not in list of allowed graphs: " + allowedGraphs); }
 
 			// upload each OWL file to model graph
 			for(String fileStr : this.getFiles()) {

sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/ManifestConfig.java

@@ -242,12 +242,14 @@ public void load(String server, String serverTypeString, boolean clear, boolean
 				// load via an owl ingestion YAML
 				File stepFile = new File(baseDir, (String)step.getValue());
 				LoadOwlConfig config = new LoadOwlConfig(stepFile, this.defaultModelGraph);
+				config.setAllowedGraphs(getGraphsFootprint());
 				config.load(null, server, serverTypeString, queryClient, logger);
 
 			}else if(type == StepType.DATA) {
 				// load content using CSV ingestion YAML
 				File stepFile = new File(baseDir, (String)step.getValue());
 				LoadDataConfig config = new LoadDataConfig(stepFile, this.defaultModelGraph, this.defaultDataGraph);
+				config.setAllowedGraphs(getGraphsFootprint());
 				config.load(null, null, server, serverTypeString, false, ingestClient, ngeClient, queryClient, logger);
 
 			}else if(type == StepType.NODEGROUPS) {
@@ -263,6 +265,9 @@ public void load(String server, String serverTypeString, boolean clear, boolean
 				// load content using sub-manifest
 				File stepFile = new File(baseDir, (String)step.getValue());
 				ManifestConfig subManifest = new ManifestConfig(stepFile, defaultModelGraph, defaultDataGraph);
+				if(!getGraphsFootprint().containsAll(subManifest.getGraphsFootprint())){  // require sub-manifest footprint to be subset of parent footprint
+					throw new Exception("Sub-manifest footprint " + subManifest.getGraphsFootprint() + " is not a subset of parent footprint " + getGraphsFootprint());
+				}
 				subManifest.load(server, serverTypeString, false, false, ingestClient, ngeClient, ngStoreClient, queryClient, logger);
 
 			}else if(type == StepType.COPYGRAPH) {

sparqlGraphLibrary/src/main/java/com/ge/research/semtk/load/config/YamlConfig.java

@@ -12,6 +12,7 @@
  */
 public abstract class YamlConfig {
 
+	protected String fileName;				// the YAML file name
 	protected String baseDir;				// the directory containing the YAML file
 	protected String defaultModelGraph;		// load to this model graph if not otherwise specified
 	protected String defaultDataGraph;		// load to this data graph if not otherwise specified
@@ -23,9 +24,10 @@ public abstract class YamlConfig {
 	 * Constructor
 	 */
 	public YamlConfig(File yamlFile, File schemaFile, String defaultModelGraph, String defaultDataGraph) throws Exception {
-		setBaseDir(yamlFile.getParent());
-		setDefaultModelGraph(defaultModelGraph);
-		setDefaultDataGraph(defaultDataGraph);
+		this.fileName = yamlFile.getName();
+		this.baseDir = yamlFile.getParent();
+		this.defaultModelGraph = defaultModelGraph;
+		this.defaultDataGraph = defaultDataGraph;
 
 		// validate manifest YAML against schema
 		String yamlStr = Utility.getStringFromFilePath(yamlFile.getAbsolutePath());
@@ -38,6 +40,9 @@ public YamlConfig(File yamlFile, File schemaFile, String defaultModelGraph, Stri
 	/**
 	 * Get methods
 	 */
+	public String getFileName() {
+		return fileName;
+	}
 	public String getBaseDir() {
 		return baseDir;
 	}
@@ -48,19 +53,6 @@ public String getDefaultDataGraph() {
 		return defaultDataGraph;
 	}
 
-	/**
-	 * Set methods
-	 */
-	protected void setBaseDir(String baseDir) {
-		this.baseDir = baseDir;
-	}
-	protected void setDefaultModelGraph(String defaultModelGraph) {
-		this.defaultModelGraph = defaultModelGraph;
-	}
-	protected void setDefaultDataGraph(String defaultDataGraph) {
-		this.defaultDataGraph = defaultDataGraph;
-	}
-
 	/**
 	 * Utility method for a node that may be a string or a string array.
 	 * Returns the string or the first array element, and errors if the array has multiple elements.

sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadDataConfigTest_IT.java

@@ -134,6 +134,25 @@ public void testLoad() throws Exception{
 			assertEquals(dataSei.getNumTriples(), 0);
 			assertEquals(dataFallbackSei.getNumTriples(), NUM_EXPECTED_TRIPLES);
 
+			// test specifying allowed graphs, where graphs to be loaded are present in the allowed graphs
+			LinkedList<String> allowedGraphs = new LinkedList<String>(Arrays.asList(modelSei.getGraph(), dataSei.getGraph()));
+			loadOwlConfig.setAllowedGraphs(allowedGraphs);
+			loadDataConfig.setAllowedGraphs(allowedGraphs);
+			loadOwlConfig.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);  // loads OWL
+			loadDataConfig.load(modelSei.getGraph(), new LinkedList<String>(Arrays.asList(dataSei.getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
+
+			// test specifying allowed graphs, where graphs to be loaded are NOT present in the allowed graphs
+			try {
+				allowedGraphs = new LinkedList<String>(Arrays.asList(modelSei.getGraph(), dataSei.getGraph() + "-123"));
+				loadOwlConfig.setAllowedGraphs(allowedGraphs);
+				loadDataConfig.setAllowedGraphs(allowedGraphs);
+				loadOwlConfig.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);  // loads OWL
+				loadDataConfig.load(modelSei.getGraph(), new LinkedList<String>(Arrays.asList(dataSei.getGraph())), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), false, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
+				fail();
+			}catch(Exception e) {
+				assertTrue(e.getMessage().contains(dataSei.getGraph() + " is not in list of allowed graphs: [" + modelSei.getGraph() + ", " + dataSei.getGraph() + "-123]"));
+			}
+
 		}finally {
 			clearGraphs();
 		}

sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/LoadOwlConfigTest_IT.java

@@ -22,8 +22,9 @@
 import static org.junit.Assert.fail;
 
 import java.io.File;
-import java.io.PrintWriter;
 import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.LinkedList;
 
 import org.junit.Test;
 
@@ -112,6 +113,19 @@ public void testLoad() throws Exception{
 			assertEquals(modelSei.getNumTriples(), 0);
 			assertEquals(modelFallbackSei.getNumTriples(), NUM_EXPECTED_TRIPLES);
 
+			// test specifying allowed graphs, where graphs to be loaded are present in the allowed graphs
+			config.setAllowedGraphs(new LinkedList<String>(Arrays.asList(modelSei.getGraph())));
+			config.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
+
+			// test specifying allowed graphs, where graphs to be loaded are NOT present in the allowed graphs
+			config.setAllowedGraphs(new LinkedList<String>(Arrays.asList(modelSei.getGraph() + "-123")));
+			try {
+				config.load(modelSei.getGraph(), TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
+				fail();
+			}catch(Exception e) {
+				assertTrue(e.getMessage().contains(modelSei.getGraph() + " is not in list of allowed graphs: [" + modelSei.getGraph() + "-123]"));
+			}
+
 		}catch(Exception e) {
 			throw e;
 		}finally {

sparqlGraphLibrary/src/test/java/com/ge/research/semtk/load/config/test/ManifestConfigTest_IT.java

@@ -17,10 +17,10 @@
 package com.ge.research.semtk.load.config.test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
-import java.io.PrintWriter;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.math3.util.Pair;
@@ -125,6 +125,52 @@ public void testLoadManifest_UsesDefaultGraph() throws Exception{
 		}
 	}
 
+	/**
+	 * Confirm error if sub-manifest footprint is not a subset of parent manifest footprint
+	 */
+	@Test
+	public void testLoadManifest_ErrorIfSubmanifestFootprintNotSubset() throws Exception{
+		File tempDir = null;
+		try {
+			tempDir = TestGraph.unzipAndUniquifyJunitGraphs(this, "/config/IngestionPackage-SubManifestFootprintNotSubset.zip");
+			ManifestConfig manifest = new ManifestConfig(ManifestConfig.getTopLevelManifestFile(tempDir), modelFallbackSei.getGraph(), dataFallbackSei.getGraph());
+			manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
+		}catch(Exception e) {
+			assertTrue(e.getMessage().contains("is not a subset of parent footprint"));
+		}finally{
+			if(tempDir != null) { FileUtils.deleteDirectory(tempDir); }
+			reset();
+		}
+	}
+
+	/**
+	 * Confirm error if try to load to a graph that is not in the footprint
+	 */
+	@Test
+	public void testLoadManifest_ErrorLoadToGraphNotInFootprint() throws Exception{
+		// Note: no copy to default graph, so don't have to limit to Fuseki only
+
+		// using generic fallbacks don't match the footprint
+		SparqlEndpointInterface modelFallbackSei_notUnique = TestGraph.getSei("http://junit/rack001/model");
+		SparqlEndpointInterface dataFallbackSei_notUnique = TestGraph.getSei("http://junit/rack001/data");
+
+		File tempDir = null;
+		try {
+			// get manifest from ingestion package, perform load
+			tempDir = TestGraph.unzipAndUniquifyJunitGraphs(this, "/config/IngestionPackage.zip");
+			ManifestConfig manifest = new ManifestConfig(ManifestConfig.getTopLevelManifestFile(tempDir), modelFallbackSei_notUnique.getGraph(), dataFallbackSei_notUnique.getGraph());
+
+			// tries loading to fallback, which is not in the footprint
+			manifest.load(TestGraph.getSparqlServer(), TestGraph.getSparqlServerType(), true, true, IntegrationTestUtility.getIngestorRestClient(), IntegrationTestUtility.getNodeGroupExecutionRestClient(), IntegrationTestUtility.getNodeGroupStoreRestClient(), IntegrationTestUtility.getSparqlQueryAuthClient(), null);
+
+		}catch(Exception e) {
+			assertTrue(e.getMessage().contains(modelFallbackSei_notUnique.getGraph() + " is not in list of allowed graphs: ["));
+		}finally{
+			if(tempDir != null) { FileUtils.deleteDirectory(tempDir); }
+			reset();
+		}
+	}
+
 	// clear graphs and nodegroup store
 	private void reset() throws Exception {
 		IntegrationTestUtility.clearGraph(modelFallbackSei);