From 8dc0871948eab8bf3c567b9278f1aa084f7f9d77 Mon Sep 17 00:00:00 2001 From: ifunga ndana Date: Sat, 5 May 2018 13:23:45 +0200 Subject: [PATCH] Added the ability to update reporting tables regardless of the new live_version table being updated or not. Renamed options. Addressed memory bug in SaveEventArgument --- build.gradle | 2 +- .../io/github/subiyacryolite/jds/JdsDb.kt | 6 +- .../io/github/subiyacryolite/jds/JdsEntity.kt | 2 +- .../io/github/subiyacryolite/jds/JdsLoad.kt | 8 +- .../github/subiyacryolite/jds/JdsOptions.kt | 15 ++-- .../io/github/subiyacryolite/jds/JdsSave.kt | 17 ++-- .../io/github/subiyacryolite/jds/JdsTable.kt | 86 ++++++++++++++----- .../jds/events/SaveEventArgument.kt | 6 +- src/test/java/common/BaseTestConfig.kt | 2 +- 9 files changed, 99 insertions(+), 45 deletions(-) diff --git a/build.gradle b/build.gradle index 2b3f028..0897732 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ sourceCompatibility = 1.8 targetCompatibility = 1.8 group = "io.github.subiyacryolite" -version = "9.1.9-SNAPSHOT" +version = "9.2.0-SNAPSHOT" archivesBaseName = project.name description = "A dynamic, cross platform, high performance, ORM data-mapper. Designed to assist in rapid development and data mining" [compileJava, compileTestJava]*.options*.encoding = "UTF-8" diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/JdsDb.kt b/src/main/kotlin/io/github/subiyacryolite/jds/JdsDb.kt index 72d671a..dcce7cb 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/JdsDb.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/JdsDb.kt @@ -36,7 +36,7 @@ import kotlin.collections.LinkedHashMap * @param implementation * @param supportsStatements */ -abstract class JdsDb(var implementation: JdsImplementation, var supportsStatements: Boolean) : IJdsDb, Serializable { +abstract class JdsDb(val implementation: JdsImplementation, val supportsStatements: Boolean) : IJdsDb, Serializable { val classes = ConcurrentHashMap>() val tables = HashSet() @@ -580,7 +580,7 @@ abstract class JdsDb(var implementation: JdsImplementation, var supportsStatemen statement.setString(3, caption) statement.setString(4, description) statement.executeUpdate() - if (options.isPrintingOutput) + if (options.isLoggingOutput) println("Mapped Entity [$name - $id]") } } catch (ex: Exception) { @@ -621,7 +621,7 @@ abstract class JdsDb(var implementation: JdsImplementation, var supportsStatemen jdsEntity.populateRefEnumRefEntityEnum(this, connection, jdsEntity.overview.entityId) mapParentEntities(connection, parentEntities, jdsEntity.overview.entityId) connection.commit() - if (options.isPrintingOutput) + if (options.isLoggingOutput) println("Mapped Entity [${entityAnnotation.name}]") } } catch (ex: Exception) { diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/JdsEntity.kt b/src/main/kotlin/io/github/subiyacryolite/jds/JdsEntity.kt index a3b7f6b..cfdae41 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/JdsEntity.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/JdsEntity.kt @@ -1150,7 +1150,7 @@ abstract class JdsEntity : IJdsEntity { internal fun populateRefEnumRefEntityEnum(jdsDb: JdsDb, connection: Connection, entityId: Long) { populateRefEnum(jdsDb, connection, getEnums(overview.entityId)) populateRefEntityEnum(jdsDb, connection, entityId, getEnums(overview.entityId)) - if (jdsDb.options.isPrintingOutput) + if (jdsDb.options.isLoggingOutput) System.out.printf("Mapped Enums for Entity[%s]\n", entityId) } diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/JdsLoad.kt b/src/main/kotlin/io/github/subiyacryolite/jds/JdsLoad.kt index a220f96..93e47ef 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/JdsLoad.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/JdsLoad.kt @@ -185,7 +185,7 @@ class JdsLoad(private val jdsDb: JdsDb, private val referenceType createEntities(entities, preparedStatement) entities.filterIsInstance(JdsLoadListener::class.java).forEach { it.onPreLoad(OnPreLoadEventArgument(jdsDb, connection, alternateConnections)) } //all entities have been initialised, now we populate them - if (jdsDb.options.isWritingToPrimaryDataTables) { + if (jdsDb.options.isWritingValuesToEavTables) { booleanStatement.use { populateBoolean(entities, it) } doubleStatement.use { populateDouble(entities, it) } enumStatement.use { populateEnum(entities, it) } @@ -194,7 +194,7 @@ class JdsLoad(private val jdsDb: JdsDb, private val referenceType longStatement.use { populateLong(entities, it) } stringStatement.use { populateString(entities, it) } } - if (jdsDb.options.isWritingToPrimaryDataTables || jdsDb.options.isWritingArrayValues) { + if (jdsDb.options.isWritingValuesToEavTables || jdsDb.options.isWritingCollectionsToEavTables) { doubleCollectionStatement.use { populateDoubleCollection(entities, it) } dateTimeCollectionStatement.use { populateDateTimeCollection(entities, it) } enumCollectionStatement.use { populateEnumCollection(entities, it) } @@ -203,7 +203,7 @@ class JdsLoad(private val jdsDb: JdsDb, private val referenceType longCollectionStatement.use { populateLongCollection(entities, it) } stringCollectionStatement.use { populateStringCollection(entities, it) } } - if (jdsDb.options.isWritingToPrimaryDataTables && jdsDb.options.initialiseDatesAndTimes) { + if (jdsDb.options.isWritingValuesToEavTables && jdsDb.options.initialiseDatesAndTimes) { dateStatement.use { populateDate(entities, it) } dateTimeStatement.use { populateDateTime(entities, it) } durationStatement.use { populateDuration(entities, it) } @@ -214,7 +214,7 @@ class JdsLoad(private val jdsDb: JdsDb, private val referenceType zonedDateTimeStatement.use { populateZonedDateTime(entities, it) } } if (jdsDb.options.initialiseObjects) { - if (jdsDb.options.isWritingToPrimaryDataTables) + if (jdsDb.options.isWritingValuesToEavTables) blobStatement.use { populateBlobs(entities, it) } populateEmbeddedAndArrayObjectsStmt.use { populateObjectEntriesAndObjectArrays(jdsDb, entities, it) } } diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/JdsOptions.kt b/src/main/kotlin/io/github/subiyacryolite/jds/JdsOptions.kt index a1da209..2f7d901 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/JdsOptions.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/JdsOptions.kt @@ -4,17 +4,17 @@ class JdsOptions { /** * A value indicating whether JDS should print internal log information */ - var isPrintingOutput: Boolean = false + var isLoggingOutput: Boolean = false /** - * Indicate whether JDS is persisting to the primary data tables + * Indicates whether JDS is persisting values to the EAV jds_str_* tables */ - var isWritingToPrimaryDataTables = true + var isWritingValuesToEavTables = true /** - * Indicate if JDS should write array types to the DB + * Indicates whether JDS is persisting collections to the EAV jds_str_*_collection tables */ - var isWritingArrayValues = true + var isWritingCollectionsToEavTables = true /** * Indicates if load operations should initialise java primitive types @@ -46,4 +46,9 @@ class JdsOptions { * property and manually call [deleteOldDataFromReportTables][JdsDb.deleteOldDataFromReportTables] from [JdsDb][JdsDb] */ var isDeletingOldDataFromReportTablesAfterSave = true + + /** + * Indicates if JDS is writing the latest version to the jds_entity_live_version table + */ + var isWritingLatestEntityVersion = true } \ No newline at end of file diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/JdsSave.kt b/src/main/kotlin/io/github/subiyacryolite/jds/JdsSave.kt index bc65473..943a12d 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/JdsSave.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/JdsSave.kt @@ -72,7 +72,7 @@ class JdsSave private constructor(private val jdsDb: JdsDb, try { val orderedList = buildSequence { batch.forEach { yieldAll(it.getNestedEntities()) } } saveInner(orderedList.asIterable(), index == (totalChunks - 1)) - if (jdsDb.options.isPrintingOutput) + if (jdsDb.options.isLoggingOutput) println("Processing saves. Batch ${index + 1} of $totalChunks") } catch (ex: Exception) { ex.printStackTrace(System.err) @@ -101,7 +101,7 @@ class JdsSave private constructor(private val jdsDb: JdsDb, //ensure that overviews are submitted before handing over to listeners saveOverview(entities) entities.forEach { - if (jdsDb.options.isWritingToPrimaryDataTables) { + if (jdsDb.options.isWritingValuesToEavTables) { saveDateConstructs(it) saveDatesAndDateTimes(it) saveZonedDateTimes(it) @@ -115,7 +115,7 @@ class JdsSave private constructor(private val jdsDb: JdsDb, saveBlobs(it) saveEnums(it) } - if (jdsDb.options.isWritingArrayValues) { + if (jdsDb.options.isWritingCollectionsToEavTables) { //array properties [NOTE arrays have old entries deleted first, for cases where a user reduced the amount of entries in the collection] saveArrayDates(it) saveArrayStrings(it) @@ -181,11 +181,12 @@ class JdsSave private constructor(private val jdsDb: JdsDb, saveLiveVersion.setString(1, it.overview.uuid) saveLiveVersion.addBatch() - updateLiveVersion.setInt(1, it.overview.editVersion) - updateLiveVersion.setString(2, it.overview.uuid) - updateLiveVersion.setInt(3, it.overview.editVersion) - updateLiveVersion.addBatch() - + if (jdsDb.options.isWritingLatestEntityVersion) { + updateLiveVersion.setInt(1, it.overview.editVersion) + updateLiveVersion.setString(2, it.overview.uuid) + updateLiveVersion.setInt(3, it.overview.editVersion) + updateLiveVersion.addBatch() + } if (jdsDb.options.isWritingToReportingTables && jdsDb.tables.isNotEmpty()) processCrt(jdsDb, postSaveEventArgument, it) diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/JdsTable.kt b/src/main/kotlin/io/github/subiyacryolite/jds/JdsTable.kt index 8a746f9..59e630b 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/JdsTable.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/JdsTable.kt @@ -25,7 +25,6 @@ import java.sql.Connection import java.sql.Timestamp import java.time.* import java.util.* -import java.util.concurrent.ConcurrentMap import kotlin.collections.HashMap import kotlin.collections.HashSet import kotlin.collections.LinkedHashMap @@ -116,7 +115,6 @@ open class JdsTable() : Serializable { /** * * @param jdsDb an instance of JdsDb, used to lookup mapped classes and determine SQL types based on implementation - * @param alternateConnections a [ConcurrentMap] of holding a pool of alternate [Connections] to write to * @param entity a [JdsEntity][JdsEntity] that may have [JdsField'S][JdsField] persisted to this [JdsTable] * @param eventArgument The [EventArgument][EventArgument] that will hold batched SQL queries for execution * @throws Exception General IO errors @@ -208,7 +206,7 @@ open class JdsTable() : Serializable { if (!jdsDb.doesTableExist(connection, name)) { connection.prepareStatement(JdsSchema.generateTable(jdsDb, name)).use { it.executeUpdate() - if (jdsDb.options.isPrintingOutput) + if (jdsDb.options.isLoggingOutput) println("Created $name") } } @@ -342,24 +340,70 @@ open class JdsTable() : Serializable { * @implNote Use the recommended style for each DB Engine to ensure optimal performance */ internal fun deleteOldRecords(jdsDb: JdsDb) = when (jdsDb.implementation) { - JdsImplementation.TSQL -> "DELETE $name FROM $name report_table\n" + - "INNER JOIN jds_entity_live_version live_records\n" + - "ON live_records.uuid = report_table.uuid\n" + - "AND live_records.edit_version = report_table.edit_version\n" + - "WHERE live_records.uuid IS NULL" - JdsImplementation.POSTGRES -> "DELETE FROM $name AS report_table\n" + - "WHERE NOT EXISTS ( SELECT * from jds_entity_live_version AS live_records\n" + - "WHERE report_table.edit_version = live_records.edit_version\n" + - "AND report_table.uuid = live_records.uuid)" - JdsImplementation.MARIADB, JdsImplementation.MYSQL -> "DELETE report_table FROM $name report_table\n" + - "LEFT JOIN jds_entity_live_version live_records ON live_records.uuid = report_table.uuid\n" + - "AND live_records.edit_version = report_table.edit_version\n" + - "WHERE live_records.uuid IS NULL" - JdsImplementation.ORACLE,JdsImplementation.SQLITE -> "DELETE FROM $name\n" + - "WHERE NOT EXISTS(SELECT *\n" + - "FROM jds_entity_live_version\n" + - "WHERE $name.uuid = jds_entity_live_version.uuid AND\n" + - "$name.edit_version = jds_entity_live_version.edit_version)" + JdsImplementation.TSQL -> { + when (jdsDb.options.isWritingLatestEntityVersion) { + true -> "DELETE $name FROM $name report_table\n" + + "INNER JOIN jds_entity_live_version live_records\n" + + "ON live_records.uuid = report_table.uuid\n" + + "AND live_records.edit_version = report_table.edit_version\n" + + "WHERE live_records.uuid IS NULL" + false -> "DELETE $name FROM $name report_table\n" + + " LEFT JOIN (SELECT\n" + + " MAX(eo.edit_version) AS edit_version,\n" + + " eo.uuid\n" + + " FROM jds_entity_overview eo\n" + + " GROUP BY eo.uuid) live_records\n" + + " ON live_records.uuid = report_table.uuid AND live_records.edit_version = report_table.edit_version\n" + + "WHERE live_records.uuid IS NULL" + } + } + JdsImplementation.POSTGRES -> { + when (jdsDb.options.isWritingLatestEntityVersion) { + true -> "DELETE FROM $name AS report_table\n" + + "WHERE NOT EXISTS ( SELECT * from jds_entity_live_version AS live_records\n" + + "WHERE report_table.edit_version = live_records.edit_version\n" + + "AND report_table.uuid = live_records.uuid)" + false -> "DELETE FROM $name AS report_table\n" + + "WHERE NOT EXISTS(SELECT\n" + + " MAX(eo.edit_version) AS edit_version,\n" + + " eo.uuid\n" + + " FROM jds_entity_overview eo\n" + + " WHERE eo.uuid = report_table.uuid AND eo.edit_version = report_table.edit_version\n" + + " GROUP BY eo.uuid)" + } + } + JdsImplementation.MARIADB, JdsImplementation.MYSQL -> { + when (jdsDb.options.isWritingLatestEntityVersion) { + true -> "DELETE report_table FROM $name report_table\n" + + "LEFT JOIN jds_entity_live_version live_records ON live_records.uuid = report_table.uuid\n" + + "AND live_records.edit_version = report_table.edit_version\n" + + "WHERE live_records.uuid IS NULL" + false -> "DELETE report_table FROM $name report_table\n" + + " LEFT JOIN (SELECT\n" + + " MAX(eo.edit_version) AS edit_version,\n" + + " eo.uuid\n" + + " FROM jds_entity_overview eo\n" + + " GROUP BY eo.uuid) live_records ON live_records.uuid = report_table.uuid\n" + + " AND live_records.edit_version = report_table.edit_version\n" + + "WHERE live_records.uuid IS NULL" + } + } + JdsImplementation.ORACLE, JdsImplementation.SQLITE -> { + when (jdsDb.options.isWritingLatestEntityVersion) { + true -> "DELETE FROM $name\n" + + "WHERE NOT EXISTS(SELECT *\n" + + "FROM jds_entity_live_version\n" + + "WHERE $name.uuid = jds_entity_live_version.uuid AND\n" + + "$name.edit_version = jds_entity_live_version.edit_version)" + false -> "DELETE FROM $name\n" + + "WHERE NOT EXISTS(SELECT\n" + + " MAX(eo.edit_version) AS edit_version,\n" + + " eo.uuid\n" + + " FROM jds_entity_overview eo\n" + + " WHERE eo.uuid = $name.uuid AND eo.edit_version = $name.edit_version\n" + + " GROUP BY eo.uuid)" + } + } } diff --git a/src/main/kotlin/io/github/subiyacryolite/jds/events/SaveEventArgument.kt b/src/main/kotlin/io/github/subiyacryolite/jds/events/SaveEventArgument.kt index fb01dc0..3efb429 100644 --- a/src/main/kotlin/io/github/subiyacryolite/jds/events/SaveEventArgument.kt +++ b/src/main/kotlin/io/github/subiyacryolite/jds/events/SaveEventArgument.kt @@ -28,11 +28,15 @@ open class SaveEventArgument(jdsDb: IJdsDb, connection: Connection, alternateCon override fun executeBatches() = try { connection.autoCommit = false alternateConnections.forEach { it.value.autoCommit = false } - statements.values.forEach { it.executeBatch() } + statements.values.forEach { + it.executeBatch() + it.close() + } connection.commit() alternateConnections.forEach { it.value.commit() } } catch (exception: Exception) { connection.rollback() + alternateConnections.forEach { it.value.rollback() } exception.printStackTrace(System.err) } finally { connection.autoCommit = true diff --git a/src/test/java/common/BaseTestConfig.kt b/src/test/java/common/BaseTestConfig.kt index 6e415f8..aa1e7d6 100644 --- a/src/test/java/common/BaseTestConfig.kt +++ b/src/test/java/common/BaseTestConfig.kt @@ -244,7 +244,7 @@ abstract class BaseTestConfig(val testName: String) { private fun initJds() { jdsDb.init() - jdsDb.options.isPrintingOutput = true + jdsDb.options.isLoggingOutput = true initialiseJdsClasses() println("=========== ${jdsDb.implementation} :: $testName ===========\n") }