Skip to content

Commit

Permalink
Merge pull request #198 from itesla/cim_uncompressed
Browse files Browse the repository at this point in the history
Import uncompressed CIM files
  • Loading branch information
geofjamg authored Jan 17, 2017
2 parents 06e184f + cdc15c7 commit 80da42d
Show file tree
Hide file tree
Showing 10 changed files with 3,746 additions and 41 deletions.
22 changes: 16 additions & 6 deletions cim1-import/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,14 @@
<name>CIM ENTSO-E V1 import</name>

<dependencies>
<!-- Compilation dependencies -->
<dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<groupId>com.google.jimfs</groupId>
<artifactId>jimfs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.servicemix.bundles</groupId>
Expand All @@ -40,16 +41,20 @@
<artifactId>jgrapht-core</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>eu.itesla_project</groupId>
<artifactId>iidm-converter-api</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<groupId>eu.itesla_project</groupId>
<artifactId>iidm-network-api</artifactId>
</dependency>

<dependency>
<groupId>eu.itesla_project</groupId>
<groupId>${project.groupId}</groupId>
<artifactId>cim1-model</artifactId>
<version>${project.version}</version>
</dependency>
Expand All @@ -60,6 +65,11 @@
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.google.common.base.Suppliers;
import com.google.gdata.util.io.base.UnicodeReader;
import eu.itesla_project.commons.config.PlatformConfig;
import eu.itesla_project.iidm.datasource.DataSourceUtil;
import eu.itesla_project.iidm.datasource.ReadOnlyDataSource;
import eu.itesla_project.iidm.import_.Importer;
import eu.itesla_project.iidm.import_.Importers;
Expand All @@ -38,7 +39,6 @@
import java.util.Properties;

/**
*
* @author Olivier Bretteville <olivier.bretteville at rte-france.com>
*/
@AutoService(Importer.class)
Expand Down Expand Up @@ -99,10 +99,10 @@ public String getComment() {
}

private Packaging detectPackaging(ReadOnlyDataSource dataSource) throws IOException {
if (dataSource.exists("_ME", "xml")) {
if (exists(dataSource, "_ME", "xml")) {
return Packaging.MERGED;
}
if (dataSource.exists("_EQ", "xml") && dataSource.exists("_TP", "xml") && dataSource.exists("_SV", "xml")) {
if (exists(dataSource, "_EQ", "xml") && exists(dataSource, "_TP", "xml") && exists(dataSource, "_SV", "xml")) {
return Packaging.SPLIT;
}
return null;
Expand Down Expand Up @@ -158,11 +158,11 @@ public boolean exists(ReadOnlyDataSource dataSource) {
if (packaging != null) {
switch (packaging) {
case MERGED:
try (InputStream is = dataSource.newInputStream("_ME", "xml")) {
try (InputStream is = newInputStream(dataSource, "_ME", "xml")) {
return isCim14(is);
}
case SPLIT:
try (InputStream eqIs = dataSource.newInputStream("_EQ", "xml")) { // just test eq file to save time
try (InputStream eqIs = newInputStream(dataSource, "_EQ", "xml")) { // just test eq file to save time
return isCim14(eqIs);
}
default:
Expand All @@ -179,7 +179,7 @@ public boolean exists(ReadOnlyDataSource dataSource) {

private CIMModel loadMergedModel(ReadOnlyDataSource dataSource, Reader bseqr, Reader bstpr) throws Exception {
CIMModel model = new CIMModel();
try (Reader mer = new UnicodeReader(dataSource.newInputStream("_ME", "xml"), null)) {
try (Reader mer = new UnicodeReader(newInputStream(dataSource, "_ME", "xml"), null)) {

long startTime2 = System.currentTimeMillis();

Expand All @@ -195,9 +195,9 @@ private CIMModel loadMergedModel(ReadOnlyDataSource dataSource, Reader bseqr, Re

private CIMModel loadSplitModel(ReadOnlyDataSource dataSource, Reader bseqr, Reader bstpr) throws Exception {
CIMModel model = new CIMModel();
try (Reader eqr = new UnicodeReader(dataSource.newInputStream("_EQ", "xml"), null);
Reader tpr = new UnicodeReader(dataSource.newInputStream("_TP", "xml"), null);
Reader svr = new UnicodeReader(dataSource.newInputStream("_SV", "xml"), null)) {
try (Reader eqr = new UnicodeReader(newInputStream(dataSource, "_EQ", "xml"), null);
Reader tpr = new UnicodeReader(newInputStream(dataSource, "_TP", "xml"), null);
Reader svr = new UnicodeReader(newInputStream(dataSource, "_SV", "xml"), null)) {

long startTime2 = System.currentTimeMillis();

Expand Down Expand Up @@ -278,7 +278,7 @@ public Network import_(ReadOnlyDataSource dataSource, Properties parameters) {
}

CIM1NamingStrategyFactory namingStrategyFactory;
try (InputStream eqIs = dataSource.newInputStream("_EQ", "xml")) { // just test eq file to save time
try (InputStream eqIs = newInputStream(dataSource, "_EQ", "xml")) { // just test eq file to save time
namingStrategyFactory = usePsseNamingStrategy && isPtiCim14(eqIs) ? new CIM1PSSENamingStrategyFactory()
: new CIM1DefaultNamingStrategyFactory();
}
Expand All @@ -301,4 +301,24 @@ public Network import_(ReadOnlyDataSource dataSource, Properties parameters) {
return network;
}

private String getBaseName(ReadOnlyDataSource dataSource) {
String basename = dataSource.getBaseName();
if (basename.endsWith("_ME") || basename.endsWith("_EQ") || basename.endsWith("_TP") || basename.endsWith("_SV")) {
basename = basename.substring(0, basename.length() - 3);
}

return basename;
}

private InputStream newInputStream(ReadOnlyDataSource dataSource, String suffix, String ext) throws IOException {
String baseName = getBaseName(dataSource);
String fileName = DataSourceUtil.getFileName(baseName, suffix, ext);
return dataSource.newInputStream(fileName);
}

private boolean exists(ReadOnlyDataSource dataSource, String suffix, String ext) throws IOException {
String baseName = getBaseName(dataSource);
String fileName = DataSourceUtil.getFileName(baseName, suffix, ext);
return dataSource.exists(fileName);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/**
* Copyright (c) 2017, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package eu.itesla_project.iidm.import_.cim1;

import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import eu.itesla_project.iidm.datasource.DataSource;
import eu.itesla_project.iidm.datasource.FileDataSource;
import eu.itesla_project.iidm.datasource.ReadOnlyDataSource;
import eu.itesla_project.iidm.datasource.ZipFileDataSource;
import org.apache.commons.io.IOUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;

/**
* @author Mathieu Bague <mathieu.bague at rte-france.com>
*/
public class CIM1ImporterTest {

private FileSystem fileSystem;

private DataSource zdsMerged;
private DataSource zdsSplit;
private DataSource fdsMerged;
private DataSource fdsUnzippedMerged;
private DataSource fdsSplit;
private DataSource fdsUnzippedSplit;

private void copyFile(DataSource dataSource, String filename) throws IOException {
try (OutputStream stream = dataSource.newOutputStream(filename, false)) {
IOUtils.copy(getClass().getResourceAsStream("/" + filename), stream);
}
}

@Before
public void setUp() throws IOException {
fileSystem = Jimfs.newFileSystem(Configuration.unix());

Path test1 = Files.createDirectory(fileSystem.getPath("test1"));
fdsMerged = new FileDataSource(test1, "ieee14bus");
fdsUnzippedMerged = new FileDataSource(test1, "ieee14bus_ME");
copyFile(fdsMerged, "ieee14bus_ME.xml");
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_EQ.xml");
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_TP.xml");

Path test2 = Files.createDirectory(fileSystem.getPath("test2"));
zdsMerged = new ZipFileDataSource(test2, "ieee14bus");
copyFile(zdsMerged, "ieee14bus_ME.xml");
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_EQ.xml");
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_TP.xml");

Path test3 = Files.createDirectory(fileSystem.getPath("test3"));
fdsSplit = new FileDataSource(test3, "ieee14bus");
fdsUnzippedSplit = new FileDataSource(test3, "ieee14bus_EQ");
copyFile(fdsSplit, "ieee14bus_EQ.xml");
copyFile(fdsSplit, "ieee14bus_TP.xml");
copyFile(fdsSplit, "ieee14bus_SV.xml");
copyFile(fdsSplit, "ENTSO-E_Boundary_Set_EU_EQ.xml");
copyFile(fdsSplit, "ENTSO-E_Boundary_Set_EU_TP.xml");

Path test4 = Files.createDirectory(fileSystem.getPath("test4"));
zdsSplit = new ZipFileDataSource(test4, "ieee14bus");
copyFile(zdsSplit, "ieee14bus_EQ.xml");
copyFile(zdsSplit, "ieee14bus_TP.xml");
copyFile(zdsSplit, "ieee14bus_SV.xml");
copyFile(zdsSplit, "ENTSO-E_Boundary_Set_EU_EQ.xml");
copyFile(zdsSplit, "ENTSO-E_Boundary_Set_EU_TP.xml");
}

@After
public void tearDown() throws IOException {
fileSystem.close();
}

@Test
public void exists() {
CIM1Importer importer = new CIM1Importer();
Assert.assertEquals(true, importer.exists(fdsMerged));
Assert.assertEquals(true, importer.exists(fdsUnzippedMerged));
Assert.assertEquals(true, importer.exists(zdsMerged));
Assert.assertEquals(true, importer.exists(fdsSplit));
Assert.assertEquals(true, importer.exists(fdsUnzippedSplit));
Assert.assertEquals(true, importer.exists(zdsSplit));
}

@Test
public void testImport() {
testImport(fdsMerged);
testImport(fdsUnzippedMerged);
testImport(fdsSplit);
testImport(fdsUnzippedSplit);
}

private void testImport(ReadOnlyDataSource dataSource) {
CIM1Importer importer = new CIM1Importer();
try {
importer.import_(dataSource, new Properties());
Assert.fail();
} catch (RuntimeException ignored) {
}
}
}
13 changes: 13 additions & 0 deletions cim1-import/src/test/resources/ENTSO-E_Boundary_Set_EU_EQ.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8" ?>
<!--
Copyright (c) 2017, RTE (http://www.rte-france.com)
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
-->
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:cim="http://iec.ch/TC57/2009/CIM-schema-cim14#"
xmlns:md="http://iec.ch/TC57/2009/schema/CIM_model_description#">
<!-- Fake ENTSO-E Boundary file (EQ file) -->
</rdf:RDF>
13 changes: 13 additions & 0 deletions cim1-import/src/test/resources/ENTSO-E_Boundary_Set_EU_TP.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8" ?>
<!--
Copyright (c) 2017, RTE (http://www.rte-france.com)
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
-->
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:cim="http://iec.ch/TC57/2009/CIM-schema-cim14#"
xmlns:md="http://iec.ch/TC57/2009/schema/CIM_model_description#">
<!-- Fake ENTSO-E Boundary file (TP file) -->
</rdf:RDF>
Loading

0 comments on commit 80da42d

Please sign in to comment.