From 11c9bdfc6f987aef48e6b9712106fa17161854bb Mon Sep 17 00:00:00 2001 From: Rob Bygrave Date: Tue, 27 Feb 2024 21:04:06 +1300 Subject: [PATCH] Squashed commit of the following: commit 20152e16891582f8ff466a93ec2cf01871824d6d Author: Rob Bygrave Date: Tue Feb 27 20:59:51 2024 +1300 #3341 Update DatabaseConfig for fluid style + javadoc commit 1c495a738493946376cad3ff75cae05edea4f882 Author: Roland Praml Date: Mon Feb 26 10:25:42 2024 +0100 Fix: Compile errors commit 574876b758791a9b66abd11e0195a0107e6b1b6d Merge: df7b04877 77634fc3e Author: Roland Praml Date: Fri Feb 23 16:16:39 2024 +0100 Merge branch 'master-rob' into FOCONIS-string-length-validation commit df7b04877d949d8a0f9fdb615fa34963712e52df Author: Rob Bygrave Date: Mon Jun 26 21:16:32 2023 +1200 #3121 BindMaxLength validation At deploy time derive a BindMaxLength property to use per BeanProperty commit 4683877e0b76e71d7f1c4562e0bf37487f8ee641 Merge: 9a4b9d4be d0020ab8c Author: Rob Bygrave Date: Fri Jun 23 16:51:34 2023 +1200 Merge branch 'string-length-validation' of github.com:FOCONIS/ebean into FOCONIS-string-length-validation commit d0020ab8ce2e5094f7f4dada24648a1730ac7bc3 Author: Roland Praml Date: Tue Jun 20 14:47:20 2023 +0200 Length validation less invasive commit 9b3b8ad46aef9dba0704173e63bc1b63e67a32da Author: Roland Praml Date: Tue Jun 20 13:03:04 2023 +0200 extended DataBind, so that it could return the last bound object commit ca3c02bc1cce66e1176b9b7bcf414ac074a25e6e Author: Roland Praml Date: Mon Jun 19 09:52:52 2023 +0200 Failing test for SqlServer --- .../java/io/ebean/DataIntegrityException.java | 9 +- .../main/java/io/ebean/DatabaseBuilder.java | 10 ++ .../java/io/ebean/config/DatabaseConfig.java | 15 +++ .../java/io/ebean/config/LengthCheck.java | 22 ++++ .../io/ebean/config/DatabaseConfigTest.java | 11 +- .../java/io/ebean/core/type/DataBinder.java | 5 + .../io/ebean/core/type/InputStreamInfo.java | 27 +++++ .../ebeaninternal/server/bind/DataBind.java | 37 ++++++- .../server/deploy/BeanDescriptorManager.java | 18 ++++ .../server/deploy/BeanProperty.java | 20 ++++ .../server/deploy/BindMaxLength.java | 77 +++++++++++++ .../deploy/meta/DeployBeanDescriptor.java | 7 +- .../deploy/meta/DeployBeanProperty.java | 19 ++++ .../test/java/org/tests/basic/TestLength.java | 101 ++++++++++++++++++ .../java/org/tests/json/TestDbJsonLength.java | 52 +++++++++ .../org/tests/model/json/EBasicJsonMap.java | 2 +- .../org/tests/model/types/SomeFileBean.java | 7 +- ebean-test/testconfig/ebean-db2.properties | 1 + ebean-test/testconfig/ebean-oracle.properties | 1 + .../testconfig/ebean-sqlserver19.properties | 1 + 20 files changed, 430 insertions(+), 12 deletions(-) create mode 100644 ebean-api/src/main/java/io/ebean/config/LengthCheck.java create mode 100644 ebean-core-type/src/main/java/io/ebean/core/type/InputStreamInfo.java create mode 100644 ebean-core/src/main/java/io/ebeaninternal/server/deploy/BindMaxLength.java create mode 100644 ebean-test/src/test/java/org/tests/basic/TestLength.java create mode 100644 ebean-test/src/test/java/org/tests/json/TestDbJsonLength.java diff --git a/ebean-api/src/main/java/io/ebean/DataIntegrityException.java b/ebean-api/src/main/java/io/ebean/DataIntegrityException.java index 495bdd3493..c9f451ee73 100644 --- a/ebean-api/src/main/java/io/ebean/DataIntegrityException.java +++ b/ebean-api/src/main/java/io/ebean/DataIntegrityException.java @@ -3,7 +3,7 @@ import jakarta.persistence.PersistenceException; /** - * Thrown when a foreign key constraint is enforced. + * Thrown when a foreign key constraint is enforced or a field is too large. */ public class DataIntegrityException extends PersistenceException { private static final long serialVersionUID = -6740171949170180970L; @@ -14,4 +14,11 @@ public class DataIntegrityException extends PersistenceException { public DataIntegrityException(String message, Throwable cause) { super(message, cause); } + + /** + * Create with message only. + */ + public DataIntegrityException(String message) { + super(message); + } } diff --git a/ebean-api/src/main/java/io/ebean/DatabaseBuilder.java b/ebean-api/src/main/java/io/ebean/DatabaseBuilder.java index bbe3afdc95..5c1c045812 100644 --- a/ebean-api/src/main/java/io/ebean/DatabaseBuilder.java +++ b/ebean-api/src/main/java/io/ebean/DatabaseBuilder.java @@ -87,6 +87,7 @@ default DatabaseBuilder alsoIf(BooleanSupplier predicate, Consumer metricNaming) { @Deprecated DatabaseBuilder setMetricNaming(Function metricNaming); + /** + * Sets the length check mode. + */ + DatabaseConfig lengthCheck(LengthCheck lengthCheck); + /** * Provides read access (getters) for the DatabaseBuilder configuration * that has been set. @@ -3103,5 +3109,9 @@ interface Settings extends DatabaseBuilder { */ Function getMetricNaming(); + /** + * Returns the length check mode. + */ + LengthCheck getLengthCheck(); } } diff --git a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java index b3ad2ef430..9fc3ee31fa 100644 --- a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java +++ b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java @@ -545,6 +545,7 @@ public class DatabaseConfig implements DatabaseBuilder.Settings { private String dumpMetricsOptions; + private LengthCheck lengthCheck = LengthCheck.OFF; private Function metricNaming = MetricNamingMatch.INSTANCE; /** @@ -2195,6 +2196,7 @@ protected void loadSettings(PropertiesWrapper p) { jdbcFetchSizeFindEach = p.getInt("jdbcFetchSizeFindEach", jdbcFetchSizeFindEach); jdbcFetchSizeFindList = p.getInt("jdbcFetchSizeFindList", jdbcFetchSizeFindList); databasePlatformName = p.get("databasePlatformName", databasePlatformName); + lengthCheck = p.getEnum(LengthCheck.class, "lengthCheck", lengthCheck); uuidVersion = p.getEnum(UuidVersion.class, "uuidVersion", uuidVersion); uuidStateFile = p.get("uuidStateFile", uuidStateFile); @@ -2571,6 +2573,19 @@ public DatabaseConfig setMetricNaming(Function metricNaming) { return this; } + /** + * Returns the length check mode. + */ + public LengthCheck getLengthCheck() { + return lengthCheck; + } + + @Override + public DatabaseConfig lengthCheck(LengthCheck lengthCheck) { + this.lengthCheck = lengthCheck; + return this; + } + public enum UuidVersion { VERSION4, VERSION1, diff --git a/ebean-api/src/main/java/io/ebean/config/LengthCheck.java b/ebean-api/src/main/java/io/ebean/config/LengthCheck.java new file mode 100644 index 0000000000..4f4abf8871 --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/LengthCheck.java @@ -0,0 +1,22 @@ +package io.ebean.config; + +/** + * Defines the length-check mode. + * + * @author Roland Praml, FOCONIS AG + */ +public enum LengthCheck { + /** + * By default, length checking is off. This means, strings/jsons and files are passed to the DB and the DB might or might not check the length. + * The DB has to check the data length. Note this is not possible for certain datatypes (e.g. clob without size) + */ + OFF, + /** + * When enabling length check, ebean validates strings/json strings and files before saving them to DB. + */ + ON, + /** + * Same as "ON", but take the UTF8-bytelength for validation. This may be useful, if you have an UTF8 based charset (default for DB2) + */ + UTF8 +} diff --git a/ebean-api/src/test/java/io/ebean/config/DatabaseConfigTest.java b/ebean-api/src/test/java/io/ebean/config/DatabaseConfigTest.java index 63bc0f2eb3..9b5afb2b09 100644 --- a/ebean-api/src/test/java/io/ebean/config/DatabaseConfigTest.java +++ b/ebean-api/src/test/java/io/ebean/config/DatabaseConfigTest.java @@ -75,6 +75,7 @@ void testLoadWithProperties() { props.setProperty("defaultServer", "false"); props.setProperty("skipDataSourceCheck", "true"); props.setProperty("readOnlyDatabase", "true"); + props.setProperty("lengthCheck", "ON"); props.setProperty("queryPlan.enable", "true"); props.setProperty("queryPlan.thresholdMicros", "10000"); @@ -95,6 +96,7 @@ void testLoadWithProperties() { assertTrue(settings.isLoadModuleInfo()); assertTrue(settings.skipDataSourceCheck()); assertTrue(settings.readOnlyDatabase()); + assertThat(settings.getLengthCheck()).isEqualTo(LengthCheck.ON); assertTrue(settings.isIdGeneratorAutomatic()); assertFalse(settings.getPlatformConfig().isCaseSensitiveCollation()); @@ -128,8 +130,11 @@ void testLoadWithProperties() { assertThat(settings.getMappingLocations()).containsExactly("classpath:/foo","bar"); - config.setPersistBatch(PersistBatch.NONE); - config.setPersistBatchOnCascade(PersistBatch.NONE); + config.persistBatch(PersistBatch.NONE) + .persistBatchOnCascade(PersistBatch.NONE) + .lengthCheck(LengthCheck.ON) + .lengthCheck(LengthCheck.UTF8); + Properties props1 = new Properties(); props1.setProperty("ebean.persistBatch", "ALL"); @@ -146,6 +151,7 @@ void testLoadWithProperties() { assertEquals(PersistBatch.ALL, settings.getPersistBatch()); assertEquals(PersistBatch.ALL, settings.getPersistBatchOnCascade()); + assertEquals(LengthCheck.UTF8, settings.getLengthCheck()); config.setEnabledL2Regions("r0,orgs"); assertEquals("r0,orgs", settings.getEnabledL2Regions()); @@ -174,6 +180,7 @@ void test_defaults() { assertEquals(600, config.getQueryPlanCapturePeriodSecs()); assertEquals(10000L, config.getQueryPlanCaptureMaxTimeMillis()); assertEquals(10, config.getQueryPlanCaptureMaxCount()); + assertThat(config.getLengthCheck()).isEqualTo(LengthCheck.OFF); config.setLoadModuleInfo(false); assertFalse(config.isAutoLoadModuleInfo()); diff --git a/ebean-core-type/src/main/java/io/ebean/core/type/DataBinder.java b/ebean-core-type/src/main/java/io/ebean/core/type/DataBinder.java index b08bb529ac..d955445b9d 100644 --- a/ebean-core-type/src/main/java/io/ebean/core/type/DataBinder.java +++ b/ebean-core-type/src/main/java/io/ebean/core/type/DataBinder.java @@ -174,4 +174,9 @@ public interface DataBinder { */ String popJson(); + /** + * Returns the last bound object (e.g. for BindValidation). Note for InputStreams you'll get an InputStreamInfo. + */ + Object popLastObject(); + } diff --git a/ebean-core-type/src/main/java/io/ebean/core/type/InputStreamInfo.java b/ebean-core-type/src/main/java/io/ebean/core/type/InputStreamInfo.java new file mode 100644 index 0000000000..137ab4da65 --- /dev/null +++ b/ebean-core-type/src/main/java/io/ebean/core/type/InputStreamInfo.java @@ -0,0 +1,27 @@ +package io.ebean.core.type; + +import java.io.InputStream; + +/** + * Helper to transports length info of DataBind.setBinaryStream(stream, length) to BindValidation + * + * @author Roland Praml, FOCONIS AG + */ +public final class InputStreamInfo { + + private final InputStream stream; + private final long length; + + public InputStreamInfo(InputStream stream, long length) { + this.stream = stream; + this.length = length; + } + + public InputStream stream() { + return stream; + } + + public long length() { + return length; + } +} diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java b/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java index 3c4c1aedb8..3b65b788fb 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java @@ -1,10 +1,14 @@ package io.ebeaninternal.server.bind; import io.ebean.core.type.DataBinder; +import io.ebean.core.type.InputStreamInfo; import io.ebeaninternal.api.CoreLog; import io.ebeaninternal.server.core.timezone.DataTimeZone; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; import java.math.BigDecimal; import java.sql.*; import java.util.ArrayList; @@ -23,6 +27,8 @@ public class DataBind implements DataBinder { protected int pos; private String json; + private Object lastObject = null; + public DataBind(DataTimeZone dataTimeZone, PreparedStatement pstmt, Connection connection) { this.dataTimeZone = dataTimeZone; this.pstmt = pstmt; @@ -65,16 +71,19 @@ public final int currentPos() { @Override public void setObject(Object value) throws SQLException { pstmt.setObject(++pos, value); + lastObject = value; } @Override public final void setObject(Object value, int sqlType) throws SQLException { pstmt.setObject(++pos, value, sqlType); + lastObject = value; } @Override public void setNull(int jdbcType) throws SQLException { pstmt.setNull(++pos, jdbcType); + lastObject = null; } @Override @@ -96,7 +105,7 @@ public final int executeUpdate() throws SQLException { } } - private void closeInputStreams() { + public void closeInputStreams() { if (inputStreams != null) { for (InputStream inputStream : inputStreams) { try { @@ -117,36 +126,43 @@ public final PreparedStatement getPstmt() { @Override public void setString(String value) throws SQLException { pstmt.setString(++pos, value); + lastObject = value; } @Override public final void setInt(int value) throws SQLException { pstmt.setInt(++pos, value); + lastObject = null; } @Override public final void setLong(long value) throws SQLException { pstmt.setLong(++pos, value); + lastObject = null; } @Override public final void setShort(short value) throws SQLException { pstmt.setShort(++pos, value); + lastObject = null; } @Override public final void setFloat(float value) throws SQLException { pstmt.setFloat(++pos, value); + lastObject = null; } @Override public final void setDouble(double value) throws SQLException { pstmt.setDouble(++pos, value); + lastObject = null; } @Override public final void setBigDecimal(BigDecimal value) throws SQLException { pstmt.setBigDecimal(++pos, value); + lastObject = null; } @Override @@ -157,6 +173,7 @@ public final void setDate(java.sql.Date value) throws SQLException { } else { pstmt.setDate(++pos, value); } + lastObject = null; } @Override @@ -167,6 +184,7 @@ public final void setTimestamp(Timestamp value) throws SQLException { } else { pstmt.setTimestamp(++pos, value); } + lastObject = null; } @Override @@ -177,26 +195,31 @@ public final void setTime(Time value) throws SQLException { } else { pstmt.setTime(++pos, value); } + lastObject = null; } @Override public void setBoolean(boolean value) throws SQLException { pstmt.setBoolean(++pos, value); + lastObject = null; } @Override public void setBytes(byte[] value) throws SQLException { pstmt.setBytes(++pos, value); + lastObject = value; } @Override public void setByte(byte value) throws SQLException { pstmt.setByte(++pos, value); + lastObject = null; } @Override public void setChar(char value) throws SQLException { pstmt.setString(++pos, String.valueOf(value)); + lastObject = null; } @Override @@ -211,21 +234,31 @@ public void setBinaryStream(InputStream inputStream, long length) throws SQLExce } inputStreams.add(inputStream); pstmt.setBinaryStream(++pos, inputStream, length); + lastObject = new InputStreamInfo(inputStream, length); } @Override public void setBlob(byte[] bytes) throws SQLException { pstmt.setBinaryStream(++pos, new ByteArrayInputStream(bytes), bytes.length); + lastObject = bytes; } @Override public void setClob(String content) throws SQLException { pstmt.setCharacterStream(++pos, new StringReader(content), content.length()); + lastObject = content; } @Override public void setArray(String arrayType, Object[] elements) throws SQLException { pstmt.setArray(++pos, connection.createArrayOf(arrayType, elements)); + lastObject = null; } + @Override + public Object popLastObject() { + Object ret = lastObject; + lastObject = null; + return ret; + } } diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptorManager.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptorManager.java index c4d179e6dc..2cee486469 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptorManager.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanDescriptorManager.java @@ -106,6 +106,7 @@ public final class BeanDescriptorManager implements BeanDescriptorMap, SpiBeanTy private final String asOfViewSuffix; private final boolean jacksonCorePresent; private final int queryPlanTTLSeconds; + private final BindMaxLength bindMaxLength; private int entityBeanCount; private List> immutableDescriptorList; /** @@ -160,6 +161,19 @@ public BeanDescriptorManager(InternalConfiguration config) { this.changeLogListener = config.changeLogListener(bootupClasses.getChangeLogListener()); this.changeLogRegister = config.changeLogRegister(bootupClasses.getChangeLogRegister()); this.jacksonCorePresent = config.isJacksonCorePresent(); + this.bindMaxLength = initMaxLength(); + } + + BindMaxLength initMaxLength() { + LengthCheck lengthCheck = this.config.getLengthCheck(); + switch (lengthCheck) { + case OFF: + return null; + case UTF8: + return BindMaxLength.ofUtf8(); + default: + return BindMaxLength.ofStandard(); + } } @Override @@ -1501,6 +1515,10 @@ public List queryPlanInit(QueryPlanInit request) { return list; } + public BindMaxLength bindMaxLength() { + return bindMaxLength; + } + /** * Comparator to sort the BeanDescriptors by name. */ diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java index 4cac21ef22..7dd2e9a33b 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java @@ -1,16 +1,20 @@ package io.ebeaninternal.server.deploy; import com.fasterxml.jackson.core.JsonToken; +import io.ebean.DataIntegrityException; import io.ebean.ValuePair; import io.ebean.bean.EntityBean; import io.ebean.bean.EntityBeanIntercept; import io.ebean.bean.MutableValueInfo; import io.ebean.bean.PersistenceContext; import io.ebean.config.EncryptKey; +import io.ebean.config.LengthCheck; import io.ebean.config.dbplatform.DbEncryptFunction; import io.ebean.config.dbplatform.DbPlatformType; +import io.ebean.config.dbplatform.ExtraDbTypes; import io.ebean.core.type.DataReader; import io.ebean.core.type.DocPropertyType; +import io.ebean.core.type.InputStreamInfo; import io.ebean.core.type.ScalarType; import io.ebean.plugin.Property; import io.ebean.text.StringParser; @@ -46,6 +50,7 @@ import java.io.DataOutput; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.sql.Types; import java.util.List; @@ -166,6 +171,7 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty { */ private final String dbComment; private final DbEncryptFunction dbEncryptFunction; + private final BindMaxLength bindMaxLength; private int deployOrder; final boolean jsonSerialize; final boolean jsonDeserialize; @@ -258,6 +264,7 @@ public BeanProperty(BeanDescriptor descriptor, DeployBeanProperty deploy) { } this.jsonSerialize = deploy.isJsonSerialize(); this.jsonDeserialize = deploy.isJsonDeserialize(); + this.bindMaxLength = deploy.bindMaxLength(); } private String tableAliasIntern(BeanDescriptor descriptor, String s, boolean dbEncrypted, String dbColumn) { @@ -345,6 +352,7 @@ protected BeanProperty(BeanProperty source, BeanPropertyOverride override) { this.elPlaceHolderEncrypted = override.replace(source.elPlaceHolderEncrypted, source.dbColumn); this.jsonSerialize = source.jsonSerialize; this.jsonDeserialize = source.jsonDeserialize; + this.bindMaxLength = source.bindMaxLength; } /** @@ -555,6 +563,18 @@ public Object readSet(DbReadContext ctx, EntityBean bean) throws SQLException { @SuppressWarnings("unchecked") public void bind(DataBind b, Object value) throws SQLException { scalarType.bind(b, value); + if (bindMaxLength != null) { + Object obj = b.popLastObject(); + long length = bindMaxLength.length(dbLength, obj); + if (length > dbLength) { + b.closeInputStreams(); + String s = String.valueOf(value); // take original bind value here. + if (s.length() > 50) { + s = s.substring(0, 47) + "..."; + } + throw new DataIntegrityException("Cannot bind value '" + s + "' (effective length=" + length + ") to column '" + dbColumn + "' (length=" + dbLength + ")"); + } + } } @SuppressWarnings(value = "unchecked") diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BindMaxLength.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BindMaxLength.java new file mode 100644 index 0000000000..51bdcfa1a6 --- /dev/null +++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BindMaxLength.java @@ -0,0 +1,77 @@ +package io.ebeaninternal.server.deploy; + +import io.ebean.core.type.InputStreamInfo; + +import java.nio.charset.StandardCharsets; + +/** + * The max length check on bind values. + */ +public interface BindMaxLength { + + /** + * Return a UTF8 based implementation. + */ + static BindMaxLength ofUtf8() { + return new UTF8(); + } + + /** + * Return a standard implementation. + */ + static BindMaxLength ofStandard() { + return new Standard(); + } + + /** + * Return the length of the object. + */ + long length(int dbLength, Object obj); + + /** + * Length check based on UTF8 bytes. + */ + final class UTF8 implements BindMaxLength { + + @Override + public long length(int dbLength, Object obj) { + if (obj instanceof String) { + String s = (String) obj; + int stringLength = s.length(); + if (stringLength > dbLength) { + return stringLength; + } else if (stringLength * 4 <= dbLength) { + return -1; + } else { + return s.getBytes(StandardCharsets.UTF_8).length; + } + + } else if (obj instanceof byte[]) { + return ((byte[]) obj).length; + } else if (obj instanceof InputStreamInfo) { + return ((InputStreamInfo) obj).length(); + } else { + return -1; + } + } + } + + /** + * Standard string length implementation. + */ + final class Standard implements BindMaxLength { + + @Override + public long length(int dbLength, Object obj) { + if (obj instanceof String) { + return ((String) obj).length(); + } else if (obj instanceof byte[]) { + return ((byte[]) obj).length; + } else if (obj instanceof InputStreamInfo) { + return ((InputStreamInfo) obj).length(); + } else { + return -1; + } + } + } +} diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java index 1f56614d1a..f96cabb465 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java @@ -1,10 +1,10 @@ package io.ebeaninternal.server.deploy.meta; +import io.ebean.DatabaseBuilder; import io.ebean.annotation.Cache; import io.ebean.annotation.DocStore; import io.ebean.annotation.DocStoreMode; import io.ebean.annotation.Identity; -import io.ebean.DatabaseBuilder; import io.ebean.config.TableName; import io.ebean.config.dbplatform.IdType; import io.ebean.config.dbplatform.PlatformIdGenerator; @@ -21,7 +21,6 @@ import io.ebeaninternal.server.idgen.UuidV1RndIdGenerator; import io.ebeaninternal.server.idgen.UuidV4IdGenerator; import io.ebeaninternal.server.rawsql.SpiRawSql; - import jakarta.persistence.Entity; import jakarta.persistence.MappedSuperclass; @@ -139,6 +138,10 @@ public DeployBeanDescriptor(BeanDescriptorManager manager, Class beanType, Da this.beanType = beanType; } + public BindMaxLength bindMaxLength() { + return manager.bindMaxLength(); + } + private String[] readPropertyNames() { try { Field field = beanType.getField("_ebean_props"); diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanProperty.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanProperty.java index 190e0b220b..b8a0102041 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanProperty.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/meta/DeployBeanProperty.java @@ -1,14 +1,17 @@ package io.ebeaninternal.server.deploy.meta; +import io.avaje.lang.Nullable; import io.ebean.annotation.*; import io.ebean.config.ScalarTypeConverter; import io.ebean.config.dbplatform.DbDefaultValue; import io.ebean.config.dbplatform.DbEncrypt; import io.ebean.config.dbplatform.DbEncryptFunction; +import io.ebean.config.dbplatform.ExtraDbTypes; import io.ebean.core.type.ScalarType; import io.ebean.util.AnnotationUtil; import io.ebeaninternal.server.core.InternString; import io.ebeaninternal.server.deploy.BeanProperty; +import io.ebeaninternal.server.deploy.BindMaxLength; import io.ebeaninternal.server.deploy.DbMigrationInfo; import io.ebeaninternal.server.deploy.DeployDocPropertyOptions; import io.ebeaninternal.server.deploy.generatedproperty.GeneratedProperty; @@ -1133,4 +1136,20 @@ boolean isJsonMapper() { boolean isJsonType() { return mutationDetection != null; } + + @Nullable + public BindMaxLength bindMaxLength() { + if (dbLength == 0) { + return null; + } + switch (dbType) { + case Types.VARCHAR: + case Types.BLOB: + case ExtraDbTypes.JSON: + return desc.bindMaxLength(); + default: + return null; + } + + } } diff --git a/ebean-test/src/test/java/org/tests/basic/TestLength.java b/ebean-test/src/test/java/org/tests/basic/TestLength.java new file mode 100644 index 0000000000..fa30635f95 --- /dev/null +++ b/ebean-test/src/test/java/org/tests/basic/TestLength.java @@ -0,0 +1,101 @@ +package org.tests.basic; + +import io.ebean.DB; +import io.ebean.DataIntegrityException; +import io.ebean.xtest.BaseTestCase; +import org.junit.jupiter.api.Test; +import org.tests.model.json.EBasicJsonList; +import org.tests.model.json.EBasicJsonMap; +import org.tests.model.types.SomeFileBean; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * @author Roland Praml, FOCONIS AG + */ +public class TestLength extends BaseTestCase { + + @Test + void testFileSize() throws IOException { + File f1 = File.createTempFile("testfile", "tmp"); + byte[] buf = new byte[1024]; + try (FileOutputStream fos = new FileOutputStream(f1)) { + for (int i = 0; i < 100; i++) { + fos.write(buf); + } + } + + SomeFileBean sfb1 = new SomeFileBean(); + sfb1.setContent(f1); + DB.save(sfb1); + + try (FileOutputStream fos = new FileOutputStream(f1)) { + for (int i = 0; i < 101; i++) { + fos.write(buf); + } + } + + SomeFileBean sfb2 = new SomeFileBean(); + sfb2.setContent(f1); + assertThatThrownBy(() -> DB.save(sfb2)).isInstanceOf(DataIntegrityException.class); + } + + + /** + * The property 'EBasicJsonMap.content' is annotated with @DbJson(length=5000). So we assume, that we cannot save Json-objects + * where the serialized form exceed that limit and we would expect an error on save. + * The length check works for platforms like h2, as H2 uses a 'varchar(5000)'. So it is impossible to save such long jsons, + * but it won't work for SqlServer, as here 'nvarchar(max)' is used. No validation happens at DB level and you might get very + * large Json objects in your database. This mostly happens unintentionally (programming error, misconfiguration) + * So they are in the database and they cannot be accessed by ebean any more, because there are new limits in Jackson: + * - Max 5 Meg per string in 2.15.0 + * - Max 20 Meg per string in 2.15.1 + * see https://github.com/FasterXML/jackson-core/issues/1014 + */ + @Test + void testLongString() { + // s is so big, that it could not be deserialized by jackson + String s = new String(new char[20_000_001]).replace('\0', 'x'); + + EBasicJsonMap bean = new EBasicJsonMap(); + bean.setName("b1"); + bean.setContent(Map.of("string", s)); + + assertThatThrownBy(() -> { + // we expect, that we can NOT save the bean, this is ensured by the bind validator. + DB.save(bean); + }).isInstanceOf(DataIntegrityException.class); + + } + + + /** + * Tests the UTF8 validation. + */ + @Test + void testUtf8() { + + String s = new String(new char[40]).replace('\0', '€'); + + EBasicJsonList bean = new EBasicJsonList(); + bean.setName("b1"); + bean.setTags(List.of(s)); + + if (isDb2() || isOracle()) { + // by default, DB2 && oracle uses bytes in varchar, so an '€' symbol needs 3 bytes + assertThatThrownBy(() -> { + // we expect, that we can NOT save the bean, this is ensured by the bind validator. + DB.save(bean); + }).isInstanceOf(DataIntegrityException.class); + } else { + DB.save(bean); + } + } + +} diff --git a/ebean-test/src/test/java/org/tests/json/TestDbJsonLength.java b/ebean-test/src/test/java/org/tests/json/TestDbJsonLength.java new file mode 100644 index 0000000000..0e35e1f563 --- /dev/null +++ b/ebean-test/src/test/java/org/tests/json/TestDbJsonLength.java @@ -0,0 +1,52 @@ +package org.tests.json; + +import io.ebean.DB; +import io.ebean.DataIntegrityException; +import org.assertj.core.api.SoftAssertions; +import org.junit.jupiter.api.Test; +import org.tests.model.json.EBasicJsonMap; + +import java.util.Map; + +class TestDbJsonLength { + + + /** + * The property 'EBasicJsonMap.content' is annotated with @DbJson(length=5000). So we assume, that we cannot save Json-objects + * where the serialized form exceed that limit and we would expect an error on save. + * The length check works for platforms like h2, as H2 uses a 'varchar(5000)'. So it is impossible to save such long jsons, + * but it won't work for SqlServer, as here 'nvarchar(max)' is used. No validation happens at DB level and you might get very + * large Json objects in your database. This mostly happens unintentionally (programming error, misconfiguration) + * So they are in the database and they cannot be accessed by ebean any more, because there are new limits in Jackson: + * - Max 5 Meg per string in 2.15.0 + * - Max 20 Meg per string in 2.15.1 + * see https://github.com/FasterXML/jackson-core/issues/1014 + */ + @Test + void testLongString() { + // s is so big, that it could not be deserialized by jackson + String s = new String(new char[20_000_001]).replace('\0', 'x'); + + EBasicJsonMap bean = new EBasicJsonMap(); + bean.setName("b1"); + bean.setContent(Map.of("string", s)); + + SoftAssertions softly = new SoftAssertions(); + + softly.assertThatThrownBy(() -> { + DB.save(bean); + }).isInstanceOf(DataIntegrityException.class); + + + if (bean.getId() != null) { + // we expect, that we could NOT save the bean, but this is not true for sqlServer. + // we will get a javax.persistence.PersistenceException: Error loading on org.tests.model.json.EBasicJsonMap.content + // when we try to load the bean back from DB + DB.find(EBasicJsonMap.class, bean.getId()); + } + + softly.assertAll(); + + } + +} diff --git a/ebean-test/src/test/java/org/tests/model/json/EBasicJsonMap.java b/ebean-test/src/test/java/org/tests/model/json/EBasicJsonMap.java index 351265f617..5feacf71cf 100644 --- a/ebean-test/src/test/java/org/tests/model/json/EBasicJsonMap.java +++ b/ebean-test/src/test/java/org/tests/model/json/EBasicJsonMap.java @@ -18,7 +18,7 @@ public class EBasicJsonMap { String name; - @DbJson + @DbJson(length = 5000) Map content; @OneToMany(mappedBy = "owner", cascade = CascadeType.ALL) diff --git a/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java b/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java index eb5f8543bd..fa6fa03bea 100644 --- a/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java +++ b/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java @@ -1,9 +1,7 @@ package org.tests.model.types; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Lob; -import jakarta.persistence.Version; +import jakarta.persistence.*; + import java.io.File; @Entity @@ -18,6 +16,7 @@ public class SomeFileBean { String name; @Lob + @Column(length = 100 * 1024) // limit to 100kb File content; public Long getId() { diff --git a/ebean-test/testconfig/ebean-db2.properties b/ebean-test/testconfig/ebean-db2.properties index 4ffab60f11..23006ac4b1 100644 --- a/ebean-test/testconfig/ebean-db2.properties +++ b/ebean-test/testconfig/ebean-db2.properties @@ -5,3 +5,4 @@ ebean.test.username=admin ebean.test.password=admin datasource.default=db2-11 ebean.db2-11.databasePlatformName=db2luw +ebean.lengthCheck=utf8 diff --git a/ebean-test/testconfig/ebean-oracle.properties b/ebean-test/testconfig/ebean-oracle.properties index 32b3af2491..388c2ec828 100644 --- a/ebean-test/testconfig/ebean-oracle.properties +++ b/ebean-test/testconfig/ebean-oracle.properties @@ -1,3 +1,4 @@ ebean.test.platform=oracle ebean.test.dbName=test_eb datasource.default=oracle +ebean.lengthCheck=utf8 diff --git a/ebean-test/testconfig/ebean-sqlserver19.properties b/ebean-test/testconfig/ebean-sqlserver19.properties index 80ffe73203..478bff7879 100644 --- a/ebean-test/testconfig/ebean-sqlserver19.properties +++ b/ebean-test/testconfig/ebean-sqlserver19.properties @@ -7,6 +7,7 @@ ebean.test.sqlserver.port=9434 ebean.test.sqlserver.url=jdbc:sqlserver://localhost:9434;databaseName=test_ebean;sendTimeAsDateTime=false;integratedSecurity=false;trustServerCertificate=true datasource.default=sqlserver2019 ebean.sqlserver2019.databasePlatformName=sqlserver17 +ebean.lengthCheck=on ## A case sensitive collation example: #ebean.test.sqlserver.collation=LATIN1_GENERAL_100_CI_AS_SC_UTF8