Skip to content

Commit

Permalink
Create a import-config module #434 #435; Added lobPrefetchSize parame…
Browse files Browse the repository at this point in the history
…ter to Oracle JDBC import module #437 [skip ci]
  • Loading branch information
hmiguim committed Feb 14, 2020
1 parent 9bec7de commit fd783b4
Show file tree
Hide file tree
Showing 90 changed files with 2,347 additions and 1,477 deletions.
58 changes: 29 additions & 29 deletions dbptk-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,10 @@
</dependency>

<!-- modules (also internal) -->
<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-dbml</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-module-dbml</artifactId>-->
<!-- </dependency>-->

<dependency>
<groupId>com.databasepreservation</groupId>
Expand All @@ -109,13 +109,13 @@

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-list-tables</artifactId>
<artifactId>dbptk-module-import-config</artifactId>
</dependency>

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-ms-access</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-module-ms-access</artifactId>-->
<!-- </dependency>-->

<dependency>
<groupId>com.databasepreservation</groupId>
Expand All @@ -127,35 +127,35 @@
<artifactId>dbptk-module-oracle</artifactId>
</dependency>

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-postgresql</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-module-postgresql</artifactId>-->
<!-- </dependency>-->

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-siard</artifactId>
</dependency>

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-sql-server</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-module-sql-server</artifactId>-->
<!-- </dependency>-->

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-sybase</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-module-sybase</artifactId>-->
<!-- </dependency>-->

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-module-openedge</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-module-openedge</artifactId>-->
<!-- </dependency>-->

<dependency>
<groupId>com.databasepreservation</groupId>
<artifactId>dbptk-filter-external-lobs</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.databasepreservation</groupId>-->
<!-- <artifactId>dbptk-filter-external-lobs</artifactId>-->
<!-- </dependency>-->

<!-- common -->
<dependency>
Expand Down
12 changes: 8 additions & 4 deletions dbptk-core/src/main/java/com/databasepreservation/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.util.Map;
import java.util.UUID;

import com.databasepreservation.model.exception.RequiredParameterException;
import org.apache.commons.cli.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -52,6 +53,7 @@ public class Main {
public static final int EXIT_CODE_CONNECTION_ERROR = 4;
public static final int EXIT_CODE_NOT_USING_UTF8 = 5;
public static final int EXIT_CODE_FILE_NOT_FOUND = 6;
public static final int EXIT_CODE_REQUIRED_PARAMETER_MISSING = 7;

private static final String execID = UUID.randomUUID().toString();

Expand Down Expand Up @@ -120,14 +122,13 @@ public static int internalMain(CLI cli) {
if (isMigrate) {
// LOGGER.info("Migrate option selected.");
exitStatus = runMigration(cli.getCLIMigrate());
}
if (isEdit) {
} else if (isEdit) {
// LOGGER.info("Edit option selected.");
exitStatus = runEdition(cli.getCLIEdit(), cli.getCLIHelp());
}
if (isValidation) {
} else if (isValidation) {
exitStatus = runValidation(cli.getCLIValidate(), cli.getCLIHelp());
}

if (exitStatus == EXIT_CODE_CONNECTION_ERROR) {
LOGGER.info("Disabling connection encryption (for modules that support it) and trying again.");
cli.disableEncryption();
Expand Down Expand Up @@ -307,6 +308,9 @@ private static int runMigration(CLIMigrate cli) {
LOGGER.info("==================================================");
logProgramFinish(EXIT_CODE_LICENSE_NOT_ACCEPTED);
return EXIT_CODE_LICENSE_NOT_ACCEPTED;
} catch (RequiredParameterException e) {
LOGGER.error("Parameter {} is required", e.getParameter());
return EXIT_CODE_REQUIRED_PARAMETER_MISSING;
} catch (ModuleException e) {
if (!e.getClass().equals(ModuleException.class)) {
LOGGER.error(e.getMessage(), e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.util.Map;
import java.util.Random;

import com.databasepreservation.model.modules.configuration.ModuleConfiguration;
import com.databasepreservation.utils.ModuleConfigurationUtils;
import org.joda.time.DateTime;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
Expand All @@ -47,7 +49,6 @@
import com.databasepreservation.model.exception.UnknownTypeException;
import com.databasepreservation.model.modules.DatabaseExportModule;
import com.databasepreservation.model.modules.DatabaseImportModule;
import com.databasepreservation.model.modules.ModuleSettings;
import com.databasepreservation.model.structure.CandidateKey;
import com.databasepreservation.model.structure.CheckConstraint;
import com.databasepreservation.model.structure.ColumnStructure;
Expand Down Expand Up @@ -566,18 +567,18 @@ protected DatabaseStructure roundtrip(DatabaseStructure dbStructure, Path tmpFil

switch (version) {
case V1_0:
exporter = new SIARD1ExportModule(tmpFile, true, false, null, null).getDatabaseHandler();
exporter = new SIARD1ExportModule(tmpFile, true, false, null).getDatabaseHandler();
break;
case V2_0:
case V2_1:
exporter = new SIARD2ExportModule(version, tmpFile, true, false, null, null).getDatabaseHandler();
exporter = new SIARD2ExportModule(version, tmpFile, true, false, null).getDatabaseHandler();
break;
case DK:
Map<String, String> exportModuleArgs = new HashMap<String, String>();
exportModuleArgs.put(SIARDDKModuleFactory.PARAMETER_FOLDER, tmpFile.toString());
exportModuleArgs.put(SIARDDKModuleFactory.PARAMETER_LOBS_PER_FOLDER, "10000");
exportModuleArgs.put(SIARDDKModuleFactory.PARAMETER_LOBS_FOLDER_SIZE, "1000");
exporter = new SIARDDKExportModule(exportModuleArgs, null).getDatabaseExportModule();
exporter = new SIARDDKExportModule(exportModuleArgs).getDatabaseExportModule();
break;
}

Expand Down Expand Up @@ -618,24 +619,24 @@ protected DatabaseStructure roundtrip(DatabaseStructure dbStructure, Path tmpFil
LOGGER.debug("SIARD file: " + tmpFile.toUri().toString());
DatabaseExportModule mocked = Mockito.mock(DatabaseExportModule.class);

Mockito.when(mocked.getModuleSettings()).thenReturn(new ModuleSettings());
Mockito.when(mocked.getModuleConfiguration()).thenReturn(ModuleConfigurationUtils.getDefaultModuleConfiguration());

DatabaseImportModule importer = null;
switch (version) {
case V1_0:
importer = new SIARD1ImportModule(tmpFile).getDatabaseImportModule();
importer = new SIARD1ImportModule(ModuleConfigurationUtils.getDefaultModuleConfiguration(), tmpFile).getDatabaseImportModule();
break;
case V2_0:
case V2_1:
importer = new SIARD2ImportModule(tmpFile).getDatabaseImportModule();
importer = new SIARD2ImportModule(ModuleConfigurationUtils.getDefaultModuleConfiguration(), tmpFile).getDatabaseImportModule();
break;

case DK:
// Notice: SIARD DK doesn't support schemas in the archive format.
// Therefore it uses a special 'importAsSchema' parameter, to make it
// compatible with the format of the dptkl internal database structure
// representation.
importer = new SIARDDKImportModule(tmpFile, dbStructure.getSchemas().get(0).getName())
importer = new SIARDDKImportModule(ModuleConfigurationUtils.getDefaultModuleConfiguration(), tmpFile, dbStructure.getSchemas().get(0).getName())
.getDatabaseImportModule();
break;
}
Expand Down
6 changes: 6 additions & 0 deletions dbptk-model/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@
<artifactId>jsch</artifactId>
<version>0.1.55</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.10.2</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<resources>
Expand Down
23 changes: 23 additions & 0 deletions dbptk-model/src/main/java/com/databasepreservation/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -150,4 +150,27 @@ public class Constants {
public static final String TRUE = "true";
public static final String FALSE = "false";

/* MODULE CONFIGURATION */
public static final String EMPTY = "";

/* CONNECTION PARAMETERS */
public static final String DB_HOST = "hostname";
public static final String DB_PORT = "port-number";
public static final String DB_USER = "username";
public static final String DB_DATABASE = "database";
public static final String DB_DISABLE_ENCRYPTION = "disable-encryption";
public static final String DB_ACCEPT_LICENSE = "accept-license";
public static final String DB_SERVER_NAME = "server-name";
public static final String DB_INSTANCE_NAME = "instance-name";
public static final String DB_INTEGRATED_LOGIN = "use-integrated-login";
public static final String DB_FILE = "file";
public static final String DB_SSH = "ssh";
public static final String DB_PASSWORD = "password";
public static final String DB_SSH_HOST = "ssh-host";
public static final String DB_SSH_PORT = "ssh-port";
public static final String DB_SSH_USER = "ssh-user";
public static final String DB_SSH_PASSWORD = "ssh-password";
public static final String DB_DRIVER = "driver";
public static final String DB_CONNECTION = "connection";

}
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ public class RemoteConnectionManager {
private static RemoteConnectionManager instance = null;
private ChannelSftp channelSftp = null;

private RemoteConnectionManager(){}
private RemoteConnectionManager() {}

public static RemoteConnectionManager getInstance(){
if(instance == null){
public static RemoteConnectionManager getInstance() {
if (instance == null) {
instance = new RemoteConnectionManager();
}
return instance;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@

import java.io.InputStream;

import org.apache.commons.compress.parallel.InputStreamSupplier;

import com.databasepreservation.model.exception.ModuleException;

/**
* @author Bruno Ferreira <bferreira@keep.pt>
*/
public interface InputStreamProvider {
public interface InputStreamProvider extends InputStreamSupplier {
/**
* Create a new input stream to read data
*
Expand All @@ -27,6 +29,15 @@ public interface InputStreamProvider {
*/
InputStream createInputStream() throws ModuleException;

@Override
default InputStream get() {
try {
return createInputStream();
} catch (ModuleException e) {
throw new RuntimeException(e.getMessage(), e);
}
}

/**
* Free all underlying resources, except for the stream itself.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package com.databasepreservation.common;

import java.io.IOException;
import java.io.InputStream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.databasepreservation.model.exception.ModuleException;

/**
* @author Miguel Guimarães <mguimaraes@keep.pt>
*/
public class InputStreamProviderImpl implements InputStreamProvider {
private static final Logger LOGGER = LoggerFactory.getLogger(InputStreamProviderImpl.class);

private InputStream inputStream;
private long size;

public InputStreamProviderImpl(InputStream inputStream) {
this.inputStream = inputStream;
}


public InputStreamProviderImpl(InputStream inputStream, long size) {
this.inputStream = inputStream;
this.size = size;
}

/**
* Create a new input stream to read data
* <p>
* Contract: The stream must be closed elsewhere. It is not closed automatically
* in any way, not even by cleanResources
*
* @return the new input stream
* @throws ModuleException
* if the input stream could not be created
*/
@Override
public InputStream createInputStream() throws ModuleException {
return inputStream;
}

/**
* Free all underlying resources, except for the stream itself.
*/
@Override
public void cleanResources() {
try {
inputStream.close();
} catch (IOException e) {
LOGGER.debug("Could not close the stream", e);
}
}

/**
* @return the total number of bytes that can be provided by the InputStream
* created with #createInputStream
* @throws ModuleException
* if the size could not be obtained
*/
@Override
public long getSize() throws ModuleException {
return size;
}
}
Loading

0 comments on commit fd783b4

Please sign in to comment.