diff --git a/R/Sql.R b/R/Sql.R index 7453de77..ad534b72 100644 --- a/R/Sql.R +++ b/R/Sql.R @@ -118,15 +118,15 @@ parseJdbcColumnData <- function(batchedQuery, column <- format(column, "%Y-%m-%d") } } else if (columnTypes[i] == 4) { - column <- rJava::.jcall(batchedQuery, "[D", "getNumeric", as.integer(i)) - column <- as.POSIXct(column, origin = "1970-01-01") - } else { - column <- rJava::.jcall(batchedQuery, "[Ljava/lang/String;", "getString", i) - if (!datesAsString) { - if (columnTypes[i] == 4) { - column <- as.POSIXct(column) - } + column <- rJava::.jcall(batchedQuery, "[Ljava/lang/String;", "getString", as.integer(i)) + if (length(column) == 0) { + timeZone <- "" + } else { + timeZone <- substr(column, 21, 999)[1] } + column <- as.POSIXct(substr(column, 1, 19), format = "%Y-%m-%d %H:%M:%OS", tz = timeZone) + } else { + column <- rJava::.jcall(batchedQuery, "[Ljava/lang/String;", "getString", as.integer(i)) } columns[[i]] <- column } diff --git a/inst/csv/jarChecksum.txt b/inst/csv/jarChecksum.txt index 41222a3b..8d96c42a 100644 --- a/inst/csv/jarChecksum.txt +++ b/inst/csv/jarChecksum.txt @@ -1 +1 @@ -c820514eb79eae56af2614eafe15d2306631218ac8738a286e4e2c878d3156de +ec60fa18e389072c979178a05dbccd921c739adf5e2a94bef5a843017f7d8631 diff --git a/inst/java/DatabaseConnector.jar b/inst/java/DatabaseConnector.jar index 38f5bb51..85949fd6 100644 Binary files a/inst/java/DatabaseConnector.jar and b/inst/java/DatabaseConnector.jar differ diff --git a/java/org/ohdsi/databaseConnector/BatchedInsert.java b/java/org/ohdsi/databaseConnector/BatchedInsert.java index 87d1ab7c..efc505af 100644 --- a/java/org/ohdsi/databaseConnector/BatchedInsert.java +++ b/java/org/ohdsi/databaseConnector/BatchedInsert.java @@ -25,7 +25,7 @@ public class BatchedInsert { public static final int BIG_DATA_BATCH_INSERT_LIMIT = 1000; - private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; + public static SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private Object[] columns; private int[] columnTypes; @@ -42,6 +42,7 @@ public BatchedInsert(Connection connection, String dbms, String sql, int columnC this.columnCount = columnCount; columns = new Object[columnCount]; columnTypes = new int[columnCount]; + // TimeZone.setDefault(TimeZone.getTimeZone("UTC")); } private void trySettingAutoCommit(boolean value) throws SQLException { @@ -103,7 +104,7 @@ private void setValue(PreparedStatement statement, int statementIndex, int rowIn if (dbms.equals(SNOWFLAKE)) setTimestampForSnowflake(statement, statementIndex, value); else - statement.setTimestamp(statementIndex, java.sql.Timestamp.valueOf(value)); + statement.setTimestamp(statementIndex, java.sql.Timestamp.valueOf(value)); } } else if (columnTypes[columnIndex] == BIGINT) { long value = ((long[]) columns[columnIndex])[rowIndex]; @@ -267,9 +268,8 @@ public void setBigint(int columnIndex, double column) { } private static void setTimestampForSnowflake(PreparedStatement statement, int statementIndex, String value) throws ParseException, SQLException { - SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT); - sdf.setTimeZone(TimeZone.getTimeZone("UTC")); - Date date = sdf.parse(value); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = dateTimeFormat.parse(value); statement.setTimestamp(statementIndex, new Timestamp(date.getTime())); } } diff --git a/java/org/ohdsi/databaseConnector/BatchedQuery.java b/java/org/ohdsi/databaseConnector/BatchedQuery.java index 686fbc4f..94db6e98 100644 --- a/java/org/ohdsi/databaseConnector/BatchedQuery.java +++ b/java/org/ohdsi/databaseConnector/BatchedQuery.java @@ -9,7 +9,8 @@ import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; -import java.util.TimeZone; +import java.text.SimpleDateFormat; +// import java.util.TimeZone; import java.sql.Date; public class BatchedQuery { @@ -26,6 +27,7 @@ public class BatchedQuery { public static double NA_DOUBLE = Double.longBitsToDouble(0x7ff00000000007a2L); public static int NA_INTEGER = Integer.MIN_VALUE; public static long NA_LONG = Long.MIN_VALUE; + public static SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); private Object[] columns; private int[] columnTypes; @@ -82,7 +84,7 @@ else if (columnTypes[columnIndex] == INTEGER64) else if (columnTypes[columnIndex] == DATE) bytesPerRow += 4; else if (columnTypes[columnIndex] == DATETIME) - bytesPerRow += 8; + bytesPerRow += 20; else // String bytesPerRow += 512; batchSize = (int) Math.min(MAX_BATCH_SIZE, Math.round((availableMemoryAtStart / 10d) / (double) bytesPerRow)); @@ -100,7 +102,7 @@ else if (columnTypes[columnIndex] == STRING) else if (columnTypes[columnIndex] == DATE) columns[columnIndex] = new int[batchSize]; else if (columnTypes[columnIndex] == DATETIME) - columns[columnIndex] = new double[batchSize]; + columns[columnIndex] = new String[batchSize]; else columns[columnIndex] = new String[batchSize]; byteBuffer = ByteBuffer.allocate(8 * batchSize); @@ -129,7 +131,7 @@ private void trySettingAutoCommit(boolean value) throws SQLException { public BatchedQuery(Connection connection, String query, String dbms) throws SQLException { this.connection = connection; this.dbms = dbms; - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + // TimeZone.setDefault(TimeZone.getTimeZone("UTC")); trySettingAutoCommit(false); Statement statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); statement.setFetchSize(FETCH_SIZE); @@ -196,9 +198,9 @@ else if (columnTypes[columnIndex] == DATE) { } else { Timestamp timestamp = resultSet.getTimestamp(columnIndex + 1); if (timestamp == null) - ((double[]) columns[columnIndex])[rowCount] = NA_DOUBLE; + ((String[]) columns[columnIndex])[rowCount] = ""; else - ((double[]) columns[columnIndex])[rowCount] = timestamp.getTime() / 1000; + ((String[]) columns[columnIndex])[rowCount] = dateTimeFormat.format(timestamp); } rowCount++; diff --git a/tests/testthat/test-insertTable.R b/tests/testthat/test-insertTable.R index fde69f64..8970962a 100644 --- a/tests/testthat/test-insertTable.R +++ b/tests/testthat/test-insertTable.R @@ -63,7 +63,7 @@ for (testServer in testServers) { ) # Check data on server is same as local - dataCopy2 <- renderTranslateQuerySql(connection, "SELECT * FROM #temp;", integer64AsNumeric = FALSE) + dataCopy2 <- renderTranslateQuerySql(connection, "SELECT some_datetime FROM #temp;", integer64AsNumeric = FALSE) names(dataCopy2) <- tolower(names(dataCopy2)) dataCopy1 <- data[order(dataCopy1$person_id), ] dataCopy2 <- dataCopy2[order(dataCopy2$person_id), ] @@ -72,6 +72,16 @@ for (testServer in testServers) { attr(dataCopy1$some_datetime, "tzone") <- NULL attr(dataCopy2$some_datetime, "tzone") <- NULL expect_equal(dataCopy1, dataCopy2, check.attributes = FALSE, tolerance = 1e-7) + # + # dataCopy1$some_datetime[1] + # dataCopy2$some_datetime[1] + # + # format(dataCopy1$some_datetime[1], format="%Y-%m-%d %H:%M:%S") + # querySql(connection, "SELECT current_timezone();") + # SqlRender::translate("SELECT TOP 1 * FROM #temp;", "spark") + # querySql(connection, "SELECT date_format(some_datetime, 'yyyy-MM-dd hh:mm:ss VV') FROM scratch.ir8uu9zutemp WHERE person_id = 2005;") + # lowLevelQuerySql(connection, "SELECT some_datetime FROM scratch.ir8uu9zutemp WHERE person_id = 2005;") + # executeSql(connection, "SET timezone = UTC;") # Check data types res <- dbSendQuery(connection, "SELECT * FROM #temp;")