Skip to content

Commit

Permalink
feat(action): sql client can handle blob (tested with h2 and oracle)
Browse files Browse the repository at this point in the history
  • Loading branch information
@KarimGl
KarimGl committed Nov 21, 2024
1 parent edf5032 commit 603d4f1
Show file tree
Hide file tree
Showing 10 changed files with 255 additions and 145 deletions.
14 changes: 14 additions & 0 deletions chutney/action-impl/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
<distribution>repo</distribution>
</license>
</licenses>
<properties>
<ojdbc11.version>23.6.0.24.10</ojdbc11.version>
</properties>

<dependencies>
<dependency>
Expand Down Expand Up @@ -300,6 +303,17 @@
<artifactId>selenium</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>oracle-free</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc11</artifactId>
<version>${ojdbc11.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
Expand Down
22 changes: 16 additions & 6 deletions chutney/action-impl/src/main/java/com/chutneytesting/action/sql/core/SqlClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,16 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SqlClient {

private final HikariDataSource dataSource;
private final int maxFetchSize;

private static final Logger LOGGER = LoggerFactory.getLogger(SqlClient.class);


public SqlClient(HikariDataSource dataSource, int maxFetchSize) {
this.dataSource = dataSource;
Expand Down Expand Up @@ -141,10 +144,10 @@ private static Object boxed(ResultSet rs, int i) throws SQLException {
return o;
}
if (o instanceof Blob) {
return new String(readBlob((Blob) o));
return readBlob((Blob) o);
}

return Optional.ofNullable(rs.getString(i)).orElse("null");
return String.valueOf(rs.getString(i));
}

private static boolean isJDBCNumericType(Class<?> type) {
Expand All @@ -167,17 +170,24 @@ private static boolean isJDBCDateType(Class<?> type) {
type.equals(Duration.class); // INTERVAL
}

private static byte[] readBlob(Blob blob) throws SQLException {
private static String readBlob(Blob blob) {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InputStream inputStream = blob.getBinaryStream()) {
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
return outputStream.toByteArray();
} catch (IOException e) {
return outputStream.toString();
} catch (IOException | SQLException e) {
throw new RuntimeException(e);
}
finally {
try {
blob.free(); // (JDBC 4.0+)
} catch (SQLException e) {
LOGGER.warn("Failed to free Blob resources: {}", e.getMessage());
}
}
}

}
Expand Down
2 changes: 1 addition & 1 deletion chutney/action-impl/src/test/java/com/chutneytesting/action/sql/SqlActionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void setUp() {
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/sql/create_db.sql", "db/sql/insert_users.sql")
.addScripts("db/common/create_users.sql")
.build();
}

Expand Down
207 changes: 127 additions & 80 deletions chutney/action-impl/src/test/java/com/chutneytesting/action/sql/core/SqlClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,111 +19,158 @@
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
import org.testcontainers.oracle.OracleContainer;
import org.testcontainers.utility.MountableFile;

public class SqlClientTest {

private static final String DB_NAME = "test_" + SqlClientTest.class;
private final Target sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:h2:mem")
.withProperty("jdbcUrl", "jdbc:h2:mem:" + DB_NAME)
.withProperty("user", "sa")
.build();

@BeforeEach
public void setUp() {
new EmbeddedDatabaseBuilder()
.setName(DB_NAME)
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/sql/create_db.sql", "db/sql/insert_users.sql", "db/sql/insert_allsqltypes.sql")
.build();
}
@Nested
class H2SqlClientTest extends AllTests {
@BeforeAll
static void beforeAll() {
sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:h2:mem")
.withProperty("jdbcUrl", "jdbc:h2:mem:" + DB_NAME)
.withProperty("user", "sa")
.build();
}

@Test
public void should_return_headers_and_rows_on_select_query() throws SQLException {
Column c0 = new Column("ID", 0);
Column c1 = new Column("NAME", 1);
Column c2 = new Column("EMAIL", 2);
@BeforeEach
public void setUp() {
new EmbeddedDatabaseBuilder()
.setName(DB_NAME)
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/common/create_users.sql", "db/h2/create_types.sql")
.build();
}

Row firstTuple = new Row(List.of(new Cell(c0, 1), new Cell(c1, "laitue"), new Cell(c2, "laitue@fake.com")));
Row secondTuple = new Row(List.of(new Cell(c0, 2), new Cell(c1, "carotte"), new Cell(c2, "kakarot@fake.db")));
Row thirdTuple = new Row(List.of(new Cell(c0, 3), new Cell(c1, "tomate"), new Cell(c2, "null")));
}

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from users");
@Nested
class OracleSqlClientTest extends AllTests {
private static OracleContainer oracle = new OracleContainer("gvenzl/oracle-free:23.4-slim-faststart")
.withDatabaseName("testDB")
.withUsername("testUser")
.withPassword("testPassword")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/oracle/init.sh"), "/container-entrypoint-initdb.d/init.sh")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/oracle/create_types.sql"), "/sql/create_types.sql")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/common/create_users.sql"), "/sql/create_users.sql");

@BeforeAll
static void beforeAll() {
oracle.start();
String address = oracle.getHost();
Integer port = oracle.getFirstMappedPort();
sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:oracle:thin:@" + address + ":" + port + "/testDB")
.withProperty("user", "testUser")
.withProperty("password", "testPassword")
.build();
}

assertThat(actual.getHeaders()).containsOnly("ID", "NAME", "EMAIL");
assertThat(actual.records).containsExactly(firstTuple, secondTuple, thirdTuple);
@AfterAll
static void afterAll() {
oracle.stop();
}
}

@Test
public void should_return_affected_rows_on_update_queries() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records records = sqlClient.execute("UPDATE USERS SET NAME = 'toto' WHERE ID = 1");
abstract static class AllTests {
protected static final String DB_NAME = "test_" + SqlClientTest.class;
protected static Target sqlTarget;

assertThat(records.affectedRows).isEqualTo(1);
}

@Test
public void should_return_count_on_count_queries() throws SQLException {
Column c0 = new Column("TOTAL", 0);
Row expectedTuple = new Row(Collections.singletonList(new Cell(c0, 3L)));
@Test
public void should_return_headers_and_rows_on_select_query() throws SQLException {

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("SELECT COUNT(*) as total FROM USERS");
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from users where ID = 1");

assertThat(actual.getHeaders()).containsOnly("ID", "NAME", "EMAIL");
assertThat(actual.records).hasSize(1);
assertThat(actual.records.get(0)).isNotNull();
List<Cell> firstRowCells = actual.records.get(0).cells;
assertThat(firstRowCells).hasSize(3);
assertThat(firstRowCells.get(0).column.name).isEqualTo("ID");
assertThat(((Number) firstRowCells.get(0).value).intValue()).isEqualTo(1);
assertThat(firstRowCells.get(1).column.name).isEqualTo("NAME");
assertThat(firstRowCells.get(1).value).isEqualTo("laitue");
assertThat(firstRowCells.get(2).column.name).isEqualTo("EMAIL");
assertThat(firstRowCells.get(2).value).isEqualTo("laitue@fake.com");
}

assertThat(actual.records).containsExactly(expectedTuple);
}
@Test
public void should_return_affected_rows_on_update_queries() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records records = sqlClient.execute("UPDATE USERS SET NAME = 'toto' WHERE ID = 1");

@Test
public void should_retrieve_columns_as_string_but_for_date_and_numeric_sql_datatypes() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from allsqltypes");

Row firstRow = actual.rows().get(0);
assertThat(firstRow.get("COL_BOOLEAN")).isInstanceOf(Boolean.class);
assertThat(firstRow.get("COL_TINYINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_SMALLINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_MEDIUMINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_INTEGER")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_BIGINT")).isInstanceOf(Long.class);
assertThat(firstRow.get("COL_FLOAT")).isInstanceOf(Float.class);
assertThat(firstRow.get("COL_DOUBLE")).isInstanceOf(Double.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DATE")).isInstanceOf(Date.class);
assertThat(firstRow.get("COL_TIME")).isInstanceOf(Time.class);
assertThat(firstRow.get("COL_TIMESTAMP")).isInstanceOf(Timestamp.class);
assertThat(firstRow.get("COL_CHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_VARCHAR")).isInstanceOf(String.class);
// INTERVAL SQL types : cf. SqlClient.StatementConverter#isJDBCDateType(Class)
assertThat(firstRow.get("COL_INTERVAL_YEAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_SECOND")).isInstanceOf(String.class);
}
assertThat(records.affectedRows).isEqualTo(1);
}

@Test
public void should_prevent_out_of_memory() {
try (MockedStatic<ChutneyMemoryInfo> chutneyMemoryInfoMockedStatic = Mockito.mockStatic(ChutneyMemoryInfo.class)) {
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::hasEnoughAvailableMemory).thenReturn(true, true, false);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::usedMemory).thenReturn(42L * 1024 * 1024);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::maxMemory).thenReturn(1337L * 1024 * 1024);
@Test
public void should_return_count_on_count_queries() throws SQLException {

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("SELECT COUNT(*) as total FROM USERS");

assertThat(actual.records).hasSize(1);
assertThat(actual.records.get(0)).isNotNull();
assertThat(actual.records.get(0).cells).hasSize(1);
assertThat(actual.records.get(0).cells.get(0)).isNotNull();
Number count = (Number) actual.records.get(0).cells.get(0).value;
assertThat(count.intValue()).isEqualTo(3);
assertThat(actual.records.get(0).cells.get(0).column.name).isEqualTo("TOTAL");
}

@Test
public void should_retrieve_columns_as_expected_datatypes() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from allsqltypes");

Row firstRow = actual.rows().get(0);
assertThat(firstRow.get("COL_BOOLEAN")).isInstanceOf(Boolean.class);
assertThat(firstRow.get("COL_INTEGER")).isInstanceOfAny(Integer.class, BigDecimal.class);
assertThat(firstRow.get("COL_FLOAT")).isInstanceOf(Float.class);
assertThat(firstRow.get("COL_DOUBLE")).isInstanceOf(Double.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DATE")).isInstanceOfAny(Date.class, Timestamp.class);
assertThat(firstRow.get("COL_TIME")).isInstanceOfAny(Time.class, String.class);
assertThat(firstRow.get("COL_TIMESTAMP")).isInstanceOfAny(Timestamp.class, String.class);
assertThat(firstRow.get("COL_CHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_VARCHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_YEAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_SECOND")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_BLOB")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_BLOB")).isEqualTo("Chutney is a funny tool.");
}

@Test
public void should_prevent_out_of_memory() {
try (MockedStatic<ChutneyMemoryInfo> chutneyMemoryInfoMockedStatic = Mockito.mockStatic(ChutneyMemoryInfo.class)) {
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::hasEnoughAvailableMemory).thenReturn(true, true, false);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::usedMemory).thenReturn(42L * 1024 * 1024);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::maxMemory).thenReturn(1337L * 1024 * 1024);

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);

Exception exception = assertThrows(NotEnoughMemoryException.class, () -> sqlClient.execute("select * from users"));
assertThat(exception.getMessage()).isEqualTo("Running step was stopped to prevent application crash. 42MB memory used of 1337MB max.\n" +
"Current step may not be the cause.\n" +
"Query fetched 2 rows");
Exception exception = assertThrows(NotEnoughMemoryException.class, () -> sqlClient.execute("select * from users"));
assertThat(exception.getMessage()).isEqualTo("Running step was stopped to prevent application crash. 42MB memory used of 1337MB max.\n" +
"Current step may not be the cause.\n" +
"Query fetched 2 rows");
}
}
}
}
7 changes: 7 additions & 0 deletions ...rc/test/resources/db/sql/insert_users.sql → ...test/resources/db/common/create_users.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@
* SPDX-License-Identifier: Apache-2.0
*
*/
DROP TABLE users;

CREATE TABLE users (
id INTEGER PRIMARY KEY,
name VARCHAR(30),
email VARCHAR(50)
);

INSERT INTO users VALUES (1, 'laitue', 'laitue@fake.com');
INSERT INTO users VALUES (2, 'carotte', 'kakarot@fake.db');
Expand Down
41 changes: 41 additions & 0 deletions chutney/action-impl/src/test/resources/db/h2/create_types.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* SPDX-FileCopyrightText: 2017-2024 Enedis
*
* SPDX-License-Identifier: Apache-2.0
*
*/

CREATE TABLE IF NOT EXISTS allsqltypes (
col_boolean BIT,
col_smallint SMALLINT,
col_integer INTEGER,
col_float REAL,
col_double DOUBLE,
col_decimal DECIMAL(20,4),
col_date DATE,
col_time TIME,
col_timestamp TIMESTAMP,
col_interval_year INTERVAL YEAR,
col_interval_second INTERVAL SECOND,
col_char CHAR,
col_varchar VARCHAR,
col_blob BLOB
);


INSERT INTO allsqltypes VALUES (
1,
66,
66666666,
666.666,
66666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666.666,
6666666666.6666,
'1966-06-06',
'06:16:16',
'1966-06-06 06:16:16',
INTERVAL '66' YEAR,
INTERVAL '6' SECOND,
'H',
'H HHH',
CAST(X'436875746e657920697320612066756e6e7920746f6f6c2e' AS BLOB)
);
Loading

0 comments on commit 603d4f1

Please sign in to comment.