Skip to content

Commit

Permalink
Fix timestamp search of database storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@jacodg
jacodg committed Dec 22, 2024
1 parent 752d536 commit b6e866d
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 26 deletions.
38 changes: 34 additions & 4 deletions src/main/java/nl/nn/testtool/MetadataExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package nl.nn.testtool;

import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
Expand All @@ -33,7 +34,27 @@ public class MetadataExtractor {
public static final int VALUE_TYPE_OBJECT = 0;
public static final int VALUE_TYPE_STRING = 1;
public static final int VALUE_TYPE_GUI = 2;
private static final DateTimeFormatter FORMAT_DATE_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withZone(ZoneId.systemDefault());
public static final String DATE_TIME_PATTERN;
public static final String DATE_TIME_RANGE_START_SUFFIX;
public static final String DATE_TIME_RANGE_END_SUFFIX;
public static final String[] DATE_TIME_RANGE_END_SPECIALS = new String[4];
static {
DATE_TIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS";
DATE_TIME_RANGE_START_SUFFIX = "0000-00-00 00:00:00.000";
DATE_TIME_RANGE_END_SUFFIX = "9999-12-31 23:59:59.999";
DATE_TIME_RANGE_END_SPECIALS[0] = " 0";
DATE_TIME_RANGE_END_SPECIALS[1] = " 1";
DATE_TIME_RANGE_END_SPECIALS[2] = " 2";
DATE_TIME_RANGE_END_SPECIALS[3] = " 1";
// Examples of special cases:
// 2024-0 should not become 2024-02 but 2024-09
// 2024-10-1 should not become 2024-10-11 but 2024-10-19
// 2024-10-2 should not become 2024-10-21 but 2024-10-29
// 2024-10-25 1 should not become 2024-10-25 13 but 2024-10-25 19
}
public static final String DATE_TIME_RANGE_END_SPECIAL2 = " 3";
public static final String DATE_TIME_RANGE_END_SPECIAL3 = " 2";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(DATE_TIME_PATTERN).withZone(ZoneId.systemDefault());
private List<MetadataFieldExtractor> metadataFieldExtractors;
/**
* Add extra fields or override standard fields (e.g. replace standard status field with application specific status
Expand Down Expand Up @@ -230,9 +251,9 @@ public String fromObjectToString(String metadataName, Object metadataValue) {

public Object fromObjectToGUI(String metadataName, Object metadataValue) {
if (metadataName.equals("startTime") || metadataName.equals("endTime")) {
return FORMAT_DATE_TIME.format(Instant.ofEpochMilli((Long)metadataValue));
return DATE_TIME_FORMATTER.format(Instant.ofEpochMilli((Long)metadataValue));
}
return metadataValue;
return fromObjectToString(metadataName, metadataValue);
}

public Object fromStringToMetadataValueType(String metadataName, String metadataValue, int metadataValueType) {
Expand Down Expand Up @@ -268,12 +289,21 @@ public Object fromStringtoObject(String metadataName, String metadataValue) {
return metadataValue;
}

public Object fromGUIToObject(String metadataName, String metadataValue) {
if (metadataName.equals("startTime") || metadataName.equals("endTime")) {
return LocalDateTime.parse(metadataValue, DATE_TIME_FORMATTER).atZone(
ZoneId.systemDefault()).toInstant().toEpochMilli();
}
return fromStringtoObject(metadataName, metadataValue);
}

public boolean isInteger(String metadataName) {
return metadataName.equals("storageId") || metadataName.equals("numberOfCheckpoints");
}

public boolean isLong(String metadataName) {
return metadataName.equals("duration") || metadataName.equals("estimatedMemoryUsage") || metadataName.equals("storageSize");
return metadataName.equals("duration") || metadataName.equals("estimatedMemoryUsage")
|| metadataName.equals("storageSize");
}

public boolean isTimestamp(String metadataName) {
Expand Down
64 changes: 42 additions & 22 deletions src/main/java/nl/nn/testtool/storage/database/DatabaseStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@
// @Dependent disabled for Quarkus for now because of the use of JdbcTemplate
public class DatabaseStorage implements Storage {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected static final String TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS";
protected @Setter @Getter String name;
protected @Setter String table;
protected @Setter @Inject @Resource(name="metadataNames") List<String> metadataNames; // Used as column names in this storage
Expand Down Expand Up @@ -434,16 +433,16 @@ protected void buildMetadataQuery(int maxNumberOfRecords, List<String> metadataN
searchValue.length() - 1);
if (StringUtils.isNotEmpty(searchValueLeft)) {
if (isInteger(column) || isLong(column)) {
addNumberExpression(query, args, argTypes, column, ">=", searchValueLeft);
addNumberExpression(query, args, argTypes, column, ">=", searchValueLeft.trim());
} else if (isTimestamp(column)) {
addTimestampExpression(query, args, argTypes, column, ">=", searchValueLeft);
addTimestampExpression(query, args, argTypes, column, ">=", searchValueLeft.trim());
}
}
if (StringUtils.isNotEmpty(searchValueRight)) {
if (isInteger(column) || isLong(column)) {
addNumberExpression(query, args, argTypes, column, "<=", searchValueRight);
addNumberExpression(query, args, argTypes, column, "<=", searchValueRight.trim());
} else if (isTimestamp(column)) {
addTimestampExpression(query, args, argTypes, column, "<=", searchValueRight);
addTimestampExpression(query, args, argTypes, column, "<=", searchValueRight.trim());
}
}
} else {
Expand All @@ -459,9 +458,9 @@ protected void buildMetadataQuery(int maxNumberOfRecords, List<String> metadataN
if (searchValue.equals("null")) {
addExpression(query, column + " is null");
} else if (isInteger(column)) {
addNumberExpression(query, args, argTypes, column, "<=", searchValue);
addNumberExpression(query, args, argTypes, column, "<=", searchValue.trim());
} else if (isTimestamp(column)) {
addTimestampExpression(query, args, argTypes, column, "<=", searchValue);
addTimestampExpression(query, args, argTypes, column, "<=", searchValue.trim());
} else {
addLikeOrEqualsExpression(query, args, argTypes, column, searchValue);
}
Expand Down Expand Up @@ -545,9 +544,19 @@ private void addTimestampExpression(StringBuilder query, List<Object> args, List
String searchValueToParse;
if (searchValue.length() < 23) {
if (">=".equals(operator)) {
searchValueToParse = searchValue + "0000-00-00T00:00:00.000".substring(searchValue.length());
searchValueToParse = searchValue
+ MetadataExtractor.DATE_TIME_RANGE_START_SUFFIX.substring(searchValue.length());
} else {
searchValueToParse = searchValue + "9999-12-31T23:59:59.999".substring(searchValue.length());
searchValueToParse = searchValue
+ MetadataExtractor.DATE_TIME_RANGE_END_SUFFIX.substring(searchValue.length());
for (int i = 0; i < MetadataExtractor.DATE_TIME_RANGE_END_SPECIALS.length; i++) {
String s = MetadataExtractor.DATE_TIME_RANGE_END_SPECIALS[i];
if (searchValue.length() == s.length()
&& searchValue.charAt(s.length() - 1) == s.charAt(s.length() - 1)) {
searchValueToParse = searchValueToParse.substring(0, s.length()) + "9"
+ searchValueToParse.substring(s.length() + 1);
}
}
}
int year = -1;
int month = -1;
Expand All @@ -566,7 +575,7 @@ private void addTimestampExpression(StringBuilder query, List<Object> args, List
if (searchValueToParse.charAt(4) != '-'
|| searchValueToParse.charAt(7) != '-'
|| searchValueToParse.charAt(7) != '-'
|| searchValueToParse.charAt(10) != 'T'
|| searchValueToParse.charAt(10) != ' '
|| searchValueToParse.charAt(13) != ':'
|| searchValueToParse.charAt(16) != ':'
|| searchValueToParse.charAt(19) != '.') {
Expand All @@ -582,14 +591,15 @@ private void addTimestampExpression(StringBuilder query, List<Object> args, List
} else {
searchValueToParse = searchValue;
}
long l = (Long)metadataExtractor.fromStringtoObject(column, searchValueToParse);
long l = (long)metadataExtractor.fromGUIToObject(column, searchValueToParse);
args.add(new Timestamp(l));
argTypes.add(Types.TIMESTAMP);
addExpression(query, column + " " + operator + " ?");
}

private void throwExceptionOnInvalidTimestamp(String searchValue) throws StorageException {
throw new StorageException("Search value '" + searchValue + "' doesn't comply with (the beginning of) pattern " + TIMESTAMP_PATTERN);
throw new StorageException("Search value '" + searchValue + "' doesn't comply with (the beginning of) pattern "
+ MetadataExtractor.DATE_TIME_PATTERN);
}

private void addExpression(StringBuilder query, String expression) {
Expand Down Expand Up @@ -627,16 +637,26 @@ public List<Object> getFilterValues(String column) throws StorageException {
}

public String getUserHelp(String column) {
String userHelpBase = SearchUtil.getUserHelpWildcards();
String userHelpBaseNonString = "Search all rows which are less than or equal to the search value.";
if (isInteger(column) || isLong(column)) {
userHelpBase = userHelpBaseNonString
+ " When the search value starts with < and ends with > a range search is done between and including the specified values separated by |.";
} else if (isTimestamp(column)) {
userHelpBase = userHelpBaseNonString
+ " When the search value only complies with the beginning of pattern yyyy-MM-dd'T'HH:mm:ss.SSS, it will be supplemented with the end of 9999-12-31T23:59:59.999";
}
return userHelpBase + SearchUtil.getUserHelpRegex() + SearchUtil.getUserHelpNullAndEmpty();
String userHelp = SearchUtil.getUserHelpWildcards();
if (isInteger(column) || isLong(column) || isTimestamp(column)) {
userHelp = "Search all rows which are";
if (isTimestamp(column)) {
userHelp = userHelp + " before";
} else {
userHelp = userHelp + " less than";
}
userHelp = userHelp + " or equal to the search value."
+ " When the search value starts with < and ends with > a range search is done between and"
+ " including the specified values separated by |.";
}
if (isTimestamp(column)) {
userHelp = userHelp
+ " When the search value only complies with the beginning of pattern "
+ MetadataExtractor.DATE_TIME_PATTERN + ", it will be supplemented with the end of "
+ MetadataExtractor.DATE_TIME_RANGE_END_SUFFIX + " or "
+ MetadataExtractor.DATE_TIME_RANGE_START_SUFFIX + " when it's the beginning of a range.";
}
return userHelp + SearchUtil.getUserHelpRegex() + SearchUtil.getUserHelpNullAndEmpty();
}

}

0 comments on commit b6e866d

Please sign in to comment.