diff --git a/src/main/java/com/teragrep/pth_06/config/ConditionConfig.java b/src/main/java/com/teragrep/pth_06/config/ConditionConfig.java new file mode 100644 index 00000000..d6e44847 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/config/ConditionConfig.java @@ -0,0 +1,53 @@ +package com.teragrep.pth_06.config; + +import com.teragrep.pth_06.planner.walker.conditions.ElementCondition; +import org.jooq.DSLContext; + +public final class ConditionConfig { + private final DSLContext ctx; + private final boolean streamQuery; + private final boolean bloomEnabled; + private final boolean withoutFilters; + + public ConditionConfig(DSLContext ctx, boolean streamQuery) { + this.ctx = ctx; + this.streamQuery = streamQuery; + this.bloomEnabled = false; + this.withoutFilters = false; + } + + public ConditionConfig(DSLContext ctx, boolean streamQuery, boolean bloomEnabled, boolean withoutFilters) { + this.ctx = ctx; + this.streamQuery = streamQuery; + this.bloomEnabled = bloomEnabled; + this.withoutFilters = withoutFilters; + } + + public DSLContext context() { + return ctx; + } + + public boolean bloomEnabled() { + return bloomEnabled; + } + + public boolean streamQuery() { + return streamQuery; + } + + public boolean withoutFilter() { + return withoutFilters; + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final ConditionConfig cast = (ConditionConfig) object; + return this.bloomEnabled == cast.bloomEnabled && + this.streamQuery == cast.streamQuery && + this.withoutFilters == cast.withoutFilters && + this.ctx == cast.ctx; + } +} \ No newline at end of file diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/ConditionWalker.java b/src/main/java/com/teragrep/pth_06/planner/walker/ConditionWalker.java index c9f8b3a6..39442e3a 100644 --- a/src/main/java/com/teragrep/pth_06/planner/walker/ConditionWalker.java +++ b/src/main/java/com/teragrep/pth_06/planner/walker/ConditionWalker.java @@ -45,29 +45,14 @@ */ package com.teragrep.pth_06.planner.walker; -import com.teragrep.blf_01.Token; -import com.teragrep.blf_01.Tokenizer; -import com.teragrep.pth_06.planner.StreamDBClient; -import org.apache.spark.util.sketch.BloomFilter; +import com.teragrep.pth_06.config.ConditionConfig; +import com.teragrep.pth_06.planner.walker.conditions.ElementCondition; import org.jooq.Condition; import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.impl.DSL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Element; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Date; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; - -import static com.teragrep.pth_06.jooq.generated.bloomdb.Bloomdb.BLOOMDB; -import static com.teragrep.pth_06.jooq.generated.journaldb.Journaldb.JOURNALDB; -import static com.teragrep.pth_06.jooq.generated.streamdb.Streamdb.STREAMDB; - /** *

Condition Walker

Walker for conditions. * @@ -76,7 +61,7 @@ * @author Mikko Kortelainen * @author Ville Manninen */ -public class ConditionWalker extends XmlWalker { +public class ConditionWalker extends XmlWalker { private final boolean bloomEnabled; private final Logger LOGGER = LoggerFactory.getLogger(ConditionWalker.class); @@ -84,22 +69,6 @@ public class ConditionWalker extends XmlWalker { private boolean streamQuery = false; private final DSLContext ctx; - final Tokenizer tokenizer; - - // TODO a hack to get global earliest value, default -24h from now - private long globalEarliestEpoch = Instant.now().getEpochSecond() - 24 * 3600; - - private void updateGlobalEarliestEpoch(long earliest) { - if (globalEarliestEpoch > earliest) { - // decrease global earliest value - globalEarliestEpoch = earliest; - } - } - - public long getGlobalEarliestEpoch() { - return globalEarliestEpoch; - } - /** * Constructor without connection. Used during unit-tests. Enables jooq-query construction. */ @@ -107,14 +76,12 @@ public ConditionWalker() { super(); this.ctx = null; this.bloomEnabled = false; - this.tokenizer = new Tokenizer(32); } public ConditionWalker(DSLContext ctx, boolean bloomEnabled) { super(); this.ctx = ctx; this.bloomEnabled = bloomEnabled; - this.tokenizer = new Tokenizer(32); } public Condition fromString(String inXml, boolean streamQuery) throws Exception { @@ -172,191 +139,8 @@ public Condition emitUnaryOperation(String op, Element current) throws Exception } Condition emitElem(Element current) { - - String tag = current.getTagName(); - - if (tag == null) { - throw new IllegalArgumentException("Tag name for Element was null"); - } - if (!current.hasAttribute("operation")) { - throw new IllegalStateException( - "Could not find specified or default value for 'operation' attribute from Element" - ); - } - if (!current.hasAttribute("value")) { - throw new IllegalStateException( - "Could not find specified or default value for 'value' attribute from Element" - ); - } - - String value = current.getAttribute("value"); - String operation = current.getAttribute("operation"); - - //System.out.println("StreamQuery="+streamQuery+" Node is terminal tag:" + tag + " val:" + value + " Operation:" + operation); - Condition queryCondition = null; - // directory - if (tag.equalsIgnoreCase("index")) { - if (streamQuery) { - queryCondition = STREAMDB.STREAM.DIRECTORY.like(value.replace('*', '%')); - } - else { - queryCondition = StreamDBClient.GetArchivedObjectsFilterTable.directory - .like(value.replace('*', '%').toLowerCase()); - } - if (operation.equalsIgnoreCase("NOT_EQUALS")) { - queryCondition = queryCondition.not(); - } - } - // stream - if (tag.equalsIgnoreCase("sourcetype")) { - if (streamQuery) { - queryCondition = STREAMDB.STREAM.STREAM_.like(value.replace('*', '%')); - } - else { - queryCondition = StreamDBClient.GetArchivedObjectsFilterTable.stream - .like(value.replace('*', '%').toLowerCase()); - } - if (operation.equalsIgnoreCase("NOT_EQUALS")) { - queryCondition = queryCondition.not(); - } - } - // host - if (tag.equalsIgnoreCase("host")) { - if (streamQuery) { - queryCondition = STREAMDB.HOST.NAME.like(value.replace('*', '%')); - } - else { - queryCondition = StreamDBClient.GetArchivedObjectsFilterTable.host - .like(value.replace('*', '%').toLowerCase()); - } - if (operation.equalsIgnoreCase("NOT_EQUALS")) { - queryCondition = queryCondition.not(); - } - } - if (!streamQuery) { - // Handle also time qualifiers - if (tag.equalsIgnoreCase("earliest") || tag.equalsIgnoreCase("index_earliest")) { - // SQL connection uses localTime in the session, so we use unix to come over the conversions - // hour based files are being used so earliest needs conversion to the point of the last hour - int earliestEpoch = Integer.parseInt(value); - - // TODO this is a hack to update globaol earliest value - updateGlobalEarliestEpoch(earliestEpoch); - - int earliestEpochHour = earliestEpoch - earliestEpoch % 3600; - Instant instant = Instant.ofEpochSecond(earliestEpochHour); - java.sql.Date timequalifier = new Date(instant.toEpochMilli()); - - queryCondition = JOURNALDB.LOGFILE.LOGDATE.greaterOrEqual(timequalifier); - /* not supported for mariadb - queryCondition = queryCondition.and(toTimestamp( - regexpReplaceAll(JOURNALDB.LOGFILE.PATH, "((^.*\\/.*-)|(\\.log\\.gz.*))", ""), - "YYYYMMDDHH24").greaterOrEqual(Timestamp.from(instant))); - */ - // NOTE uses literal path - queryCondition = queryCondition - .and( - "UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H'))" - + " >= " + instant.getEpochSecond() - ); - } - if (tag.equalsIgnoreCase("latest") || tag.equalsIgnoreCase("index_latest")) { - // SQL connection uses localTime in the session, so we use unix to come over the conversions - Instant instant = Instant.ofEpochSecond(Integer.parseInt(value)); - java.sql.Date timequalifier = new Date(instant.toEpochMilli()); - - queryCondition = JOURNALDB.LOGFILE.LOGDATE.lessOrEqual(timequalifier); - /* not supported for mariadb - queryCondition = queryCondition.and(toTimestamp( - regexpReplaceAll(JOURNALDB.LOGFILE.PATH, "((^.*\\/.*-)|(\\.log\\.gz.*))", ""), - "YYYYMMDDHH24").lessOrEqual(Timestamp.from(instant))); - */ - // NOTE uses literal path - /* - to match - 2021/09-27/sc-99-99-14-244/messages/messages-2021092722.gz.4 - 2018/04-29/sc-99-99-14-245/f17/f17.logGLOB-2018042900.log.gz - */ - - queryCondition = queryCondition - .and( - "UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H'))" - + " <= " + instant.getEpochSecond() - ); - } - // value search - if ("indexstatement".equalsIgnoreCase(tag) && bloomEnabled) { - if ("EQUALS".equals(operation)) { - - final Set tokenSet = new HashSet<>( - tokenizer.tokenize(new ByteArrayInputStream(value.getBytes(StandardCharsets.UTF_8))) - ); - - LOGGER.info("BloomFilter tokenSet <[{}]>", tokenSet.toString()); - - final BloomFilter smallFilter = BloomFilter.create(100000, 0.01); - final BloomFilter mediumFilter = BloomFilter.create(1000000, 0.03); - final BloomFilter largeFilter = BloomFilter.create(2500000, 0.05); - - tokenSet.forEach(token -> { - smallFilter.put(token.toString()); - mediumFilter.put(token.toString()); - largeFilter.put(token.toString()); - }); - - long rowId = StreamDBClient.BloomFiltersTempTable - .insert(ctx, smallFilter, mediumFilter, largeFilter); - - Condition rowIdCondition = StreamDBClient.BloomFiltersTempTable.id.eq(rowId); - - Field smallColumn = DSL - .select(StreamDBClient.BloomFiltersTempTable.fe100kfp001) - .from(StreamDBClient.BloomFiltersTempTable.BLOOM_TABLE) - .where(rowIdCondition) - .asField(); - Field mediumColumn = DSL - .select(StreamDBClient.BloomFiltersTempTable.fe1000kfpp003) - .from(StreamDBClient.BloomFiltersTempTable.BLOOM_TABLE) - .where(rowIdCondition) - .asField(); - Field largeColumn = DSL - .select(StreamDBClient.BloomFiltersTempTable.fe2500kfpp005) - .from(StreamDBClient.BloomFiltersTempTable.BLOOM_TABLE) - .where(rowIdCondition) - .asField(); - - final Field fe100kfp001 = DSL - .function( - "bloommatch", Boolean.class, smallColumn, - BLOOMDB.FILTER_EXPECTED_100000_FPP_001.FILTER - ); - final Field fe1000kfpp003 = DSL - .function( - "bloommatch", Boolean.class, mediumColumn, - BLOOMDB.FILTER_EXPECTED_1000000_FPP_003.FILTER - ); - final Field fe2500kfpp005 = DSL - .function( - "bloommatch", Boolean.class, largeColumn, - BLOOMDB.FILTER_EXPECTED_2500000_FPP_005.FILTER - ); - - Condition noBloomFilter = BLOOMDB.FILTER_EXPECTED_100000_FPP_001.FILTER - .isNull() - .and( - BLOOMDB.FILTER_EXPECTED_1000000_FPP_003.FILTER - .isNull() - .and(BLOOMDB.FILTER_EXPECTED_2500000_FPP_005.FILTER.isNull()) - ); - queryCondition = fe100kfp001 - .eq(true) - .or(fe1000kfpp003.eq(true).or(fe2500kfpp005.eq(true).or(noBloomFilter))); - LOGGER.trace("ConditionWalker.emitElement bloomCondition part <{}>", queryCondition); - } - } - } - - return queryCondition; + ElementCondition elementCondition = new ElementCondition(current, + new ConditionConfig(ctx, streamQuery, bloomEnabled, false)); + return elementCondition.condition(); } - } diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/EarliestWalker.java b/src/main/java/com/teragrep/pth_06/planner/walker/EarliestWalker.java index 728ad109..a20c5c66 100644 --- a/src/main/java/com/teragrep/pth_06/planner/walker/EarliestWalker.java +++ b/src/main/java/com/teragrep/pth_06/planner/walker/EarliestWalker.java @@ -57,7 +57,7 @@ * @author Mikko Kortelainen * @author Ville Manninen */ -public class EarliestWalker extends XmlWalker { +public class EarliestWalker extends XmlWalker { private final Logger LOGGER = LoggerFactory.getLogger(EarliestWalker.class); diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/KafkaWalker.java b/src/main/java/com/teragrep/pth_06/planner/walker/KafkaWalker.java index a0d328b7..8533f2bf 100644 --- a/src/main/java/com/teragrep/pth_06/planner/walker/KafkaWalker.java +++ b/src/main/java/com/teragrep/pth_06/planner/walker/KafkaWalker.java @@ -54,7 +54,7 @@ * @author Mikko Kortelainen * @since 08/06/2022 */ -public class KafkaWalker extends XmlWalker { +public class KafkaWalker extends XmlWalker { @Override String emitElem(Element current) { diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/XmlWalker.java b/src/main/java/com/teragrep/pth_06/planner/walker/XmlWalker.java index 4cd8313e..dce9a1fc 100644 --- a/src/main/java/com/teragrep/pth_06/planner/walker/XmlWalker.java +++ b/src/main/java/com/teragrep/pth_06/planner/walker/XmlWalker.java @@ -70,7 +70,7 @@ * @author Kimmo Leppinen * @author Mikko Kortelainen */ -public abstract class XmlWalker { +public abstract class XmlWalker { private final Logger LOGGER = LoggerFactory.getLogger(XmlWalker.class); @@ -81,8 +81,8 @@ public XmlWalker() { } - public T fromString(String inXml) throws Exception { - Object rv; + public T fromString(String inXml) throws Exception { + T rv; DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder loader = factory.newDocumentBuilder(); Document document = loader.parse(new InputSource(new StringReader(inXml))); @@ -92,25 +92,22 @@ public T fromString(String inXml) throws Exception { TreeWalker walker = traversal .createTreeWalker(document.getDocumentElement(), NodeFilter.SHOW_ELEMENT, null, true); - rv = traverse(walker, (String) null); - // if (rv != null) { - // System.out.println("XmlWalker.fromString() value:" + ((T) rv).toString()); - // } - return (T) rv; + rv = traverse(walker, null); + return rv; } /** * Walk through tree using depth-first order and generate spark-query using appropriate emit-methods. - * + * * @param walker * @param op operation String * @return Class which expr-part contains actual catalyst query tree * @throws Exception */ - public T traverse(TreeWalker walker, String op) throws Exception { + public T traverse(TreeWalker walker, String op) throws Exception { Node parend = walker.getCurrentNode(); Element current = ((Element) parend); - Object rv = null; + T rv = null; LOGGER.debug(" traverse incoming:" + current.getTagName()); try { @@ -129,7 +126,7 @@ public T traverse(TreeWalker walker, String op) throws Exception { } // get left and right Node left = walker.firstChild(); - Object lft; + T lft; switch (count) { case 1: { LOGGER.debug(" 1 child incoming:" + current + " left=" + left + " op:" + op); @@ -143,7 +140,7 @@ public T traverse(TreeWalker walker, String op) throws Exception { } case 2: { lft = traverse(walker, op); - Object rht = null; + T rht = null; Node right = walker.nextSibling(); walker.setCurrentNode(left); // System.out.println("traverse right:"+right); @@ -192,28 +189,25 @@ else if (rht == null) { /** * Abstract method which is called during traverse. Emits appropriate element * - * @param returned class * @param current DOM-element * @return correct query according to implementation class */ - abstract T emitElem(Element current); + abstract T emitElem(Element current); /** * Abstract method which is called during traverse. Emits appropriate logical operation - * - * @param returned class + * * @return correct query according to implementation class */ - abstract T emitLogicalOperation(String op, Object left, Object right) throws Exception; + abstract T emitLogicalOperation(String op, Object left, Object right) throws Exception; /** * Abstract method which is called during traverse. Emits appropriate unary operation - * - * @param returned class + * * @param current DOM-element * @return correct query according to implementation class */ - abstract T emitUnaryOperation(String op, Element current) throws Exception; + abstract T emitUnaryOperation(String op, Element current) throws Exception; // escape special chars inside value /** diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/EarliestCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/EarliestCondition.java new file mode 100644 index 00000000..919f0697 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/EarliestCondition.java @@ -0,0 +1,93 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; + +import java.sql.Date; +import java.time.Instant; + +import static com.teragrep.pth_06.jooq.generated.journaldb.Journaldb.JOURNALDB; + +public final class EarliestCondition implements QueryCondition { + private final String value; + + public EarliestCondition(String value) { + this.value = value; + } + + public Condition condition() { + // SQL connection uses localTime in the session, so we use unix to come over the conversions + // hour based files are being used so earliest needs conversion to the point of the last hour + final int earliestFromElement = Integer.parseInt(value); + final int earliestEpochHour = earliestFromElement - earliestFromElement % 3600; + final Instant instant = Instant.ofEpochSecond(earliestEpochHour); + final java.sql.Date timeQualifier = new Date(instant.toEpochMilli()); + Condition condition; + condition = JOURNALDB.LOGFILE.LOGDATE.greaterOrEqual(timeQualifier); + condition = condition.and("UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H'))" + + " >= " + instant.getEpochSecond()); + // raw SQL used here since following not supported for mariadb: + // queryCondition = queryCondition.and(toTimestamp( + // regexpReplaceAll(JOURNALDB.LOGFILE.PATH, "((^.*\\/.*-)|(\\.log\\.gz.*))", ""), + // "YYYYMMDDHH24").lessOrEqual(Timestamp.from(instant))); + // to match + // 2021/09-27/sc-99-99-14-244/messages/messages-2021092722.gz.4 + // 2018/04-29/sc-99-99-14-245/f17/f17.logGLOB-2018042900.log.gz + // NOTE uses literal path + return condition; + } + + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final EarliestCondition cast = (EarliestCondition) object; + return this.value.equals(cast.value); + } +} diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/ElementCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/ElementCondition.java new file mode 100644 index 00000000..da13911f --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/ElementCondition.java @@ -0,0 +1,138 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.blf_01.Tokenizer; +import com.teragrep.pth_06.config.ConditionConfig; +import org.jooq.Condition; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +/** + * Creates a query condition from provided dom element + */ +public final class ElementCondition { + private static final Logger LOGGER = LoggerFactory.getLogger(ElementCondition.class); + + private final Element element; + private final ConditionConfig config; + + public ElementCondition(Element element, ConditionConfig config) { + this.element = element; + this.config = config; + } + + private void validate(Element element) { + if (element.getTagName() == null) { + throw new IllegalStateException("Tag name for Element was null"); + } + if (!element.hasAttribute("operation")) { + throw new IllegalStateException("Could not find specified or default value for 'operation' attribute from Element"); + } + if (!element.hasAttribute("value")) { + throw new IllegalStateException("Could not find specified or default value for 'value' attribute from Element"); + } + } + + public Condition condition() { + validate(element); + final String tag = element.getTagName(); + final String value = element.getAttribute("value"); + final String operation = element.getAttribute("operation"); + Condition condition = DSL.noCondition(); + switch (tag.toLowerCase()) { + case "index": + QueryCondition index = new IndexCondition(value, operation, config.streamQuery()); + condition = index.condition(); + break; + case "sourcetype": + QueryCondition sourceType = new SourceTypeCondition(value, operation, config.streamQuery()); + condition = sourceType.condition(); + break; + case "host": + QueryCondition host = new HostCondition(value, operation, config.streamQuery()); + condition = host.condition(); + break; + } + if (!config.streamQuery()) { + // Handle also time qualifiers + if ("earliest".equalsIgnoreCase(tag) || "index_earliest".equalsIgnoreCase(tag)) { + QueryCondition earliest = new EarliestCondition(value); + condition = earliest.condition(); + } + if ("latest".equalsIgnoreCase(tag) || "index_latest".equalsIgnoreCase(tag)) { + QueryCondition latest = new LatestCondition(value); + condition = latest.condition(); + } + // value search + if ("indexstatement".equalsIgnoreCase(tag) && "EQUALS".equals(operation) && config.bloomEnabled()) { + IndexStatementCondition indexStatementCondition = + new IndexStatementCondition(value, config, new Tokenizer(32)); + condition = indexStatementCondition.condition(); + } + } + if (condition.equals(DSL.noCondition())) { + throw new IllegalStateException("Unsupported Element tag " + tag); + } + LOGGER.debug("Query condition: <{}>", condition); + return condition; + } + + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final ElementCondition cast = (ElementCondition) object; + boolean equalName = this.element.getTagName().equals(cast.element.getTagName()); + boolean equalOperation = this.element.getAttribute("operation").equals(cast.element.getAttribute("operation")); + boolean equalValue = this.element.getAttribute("value").equals(cast.element.getAttribute("value")); + return equalName && equalOperation && equalValue && this.config.equals(cast.config); + } +} diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/HostCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/HostCondition.java new file mode 100644 index 00000000..5d7c7b84 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/HostCondition.java @@ -0,0 +1,93 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.pth_06.planner.StreamDBClient; +import org.jooq.Condition; + +import static com.teragrep.pth_06.jooq.generated.streamdb.Streamdb.STREAMDB; + +public final class HostCondition implements QueryCondition { + private final String value; + private final String operation; + private final boolean streamQuery; + + public HostCondition(String value, String operation, boolean streamQuery) { + this.streamQuery = streamQuery; + this.value = value; + this.operation = operation; + } + + public Condition condition() { + Condition condition; + if (streamQuery) { + condition = STREAMDB.HOST.NAME.like( + value.replace('*', '%') + ); + } else { + condition = + StreamDBClient.GetArchivedObjectsFilterTable.host.like( + value.replace('*', '%').toLowerCase() + ); + } + if (operation.equalsIgnoreCase("NOT_EQUALS")) { + condition = condition.not(); + } + return condition; + } + + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final HostCondition cast = (HostCondition) object; + return this.streamQuery == cast.streamQuery && + this.value.equals(cast.value) && + this.operation.equals(cast.operation); + } +} \ No newline at end of file diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/IndexCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/IndexCondition.java new file mode 100644 index 00000000..76f94393 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/IndexCondition.java @@ -0,0 +1,94 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.pth_06.planner.StreamDBClient; +import org.jooq.Condition; + +import static com.teragrep.pth_06.jooq.generated.streamdb.Streamdb.STREAMDB; + +public final class IndexCondition implements QueryCondition { + private final String value; + private final String operation; + private final boolean streamQuery; + + public IndexCondition(String value, String operation, boolean streamQuery) { + this.streamQuery = streamQuery; + this.value = value; + this.operation = operation; + } + + public Condition condition() { + Condition condition; + if (streamQuery) { + condition = + STREAMDB.STREAM.DIRECTORY.like( + value.replace('*', '%') + ); + } else { + condition = + StreamDBClient.GetArchivedObjectsFilterTable.directory.like( + value.replace('*', '%').toLowerCase() + ); + } + if (operation.equalsIgnoreCase("NOT_EQUALS")) { + condition = condition.not(); + } + return condition; + } + + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final IndexCondition cast = (IndexCondition) object; + return this.streamQuery == cast.streamQuery && + this.value.equals(cast.value) && + this.operation.equals(cast.operation); + } +} diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/IndexStatementCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/IndexStatementCondition.java new file mode 100644 index 00000000..3b4c2b17 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/IndexStatementCondition.java @@ -0,0 +1,164 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.blf_01.Token; +import com.teragrep.blf_01.Tokenizer; +import com.teragrep.pth_06.config.ConditionConfig; +import com.teragrep.pth_06.planner.StreamDBClient; +import org.apache.spark.util.sketch.BloomFilter; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; + +import static com.teragrep.pth_06.jooq.generated.bloomdb.Bloomdb.BLOOMDB; + + +public final class IndexStatementCondition implements QueryCondition { + private final Logger LOGGER = LoggerFactory.getLogger(IndexStatementCondition.class); + + private final String value; + private final ConditionConfig config; + private final Tokenizer tokenizer; + + public IndexStatementCondition(String value, ConditionConfig config, Tokenizer tokenizer) { + this.value = value; + this.config = config; + this.tokenizer = tokenizer; + } + + public Condition condition() { + final Set tokenSet = new HashSet<>( + tokenizer.tokenize(new ByteArrayInputStream(value.getBytes(StandardCharsets.UTF_8))) + ); + + LOGGER.info("BloomFilter tokenSet <[{}]>", tokenSet); + + final BloomFilter smallFilter = BloomFilter.create(100000, 0.01); + final BloomFilter mediumFilter = BloomFilter.create(1000000, 0.03); + final BloomFilter largeFilter = BloomFilter.create(2500000, 0.05); + + tokenSet.forEach(token -> { + smallFilter.put(token.toString()); + mediumFilter.put(token.toString()); + largeFilter.put(token.toString()); + }); + + final long rowId = StreamDBClient.BloomFiltersTempTable + .insert(config.context(), smallFilter, mediumFilter, largeFilter); + + final Condition rowIdCondition = StreamDBClient.BloomFiltersTempTable.id.eq(rowId); + + final Field smallColumn = DSL + .select(StreamDBClient.BloomFiltersTempTable.fe100kfp001) + .from(StreamDBClient.BloomFiltersTempTable.BLOOM_TABLE) + .where(rowIdCondition) + .asField(); + final Field mediumColumn = DSL + .select(StreamDBClient.BloomFiltersTempTable.fe1000kfpp003) + .from(StreamDBClient.BloomFiltersTempTable.BLOOM_TABLE) + .where(rowIdCondition) + .asField(); + final Field largeColumn = DSL + .select(StreamDBClient.BloomFiltersTempTable.fe2500kfpp005) + .from(StreamDBClient.BloomFiltersTempTable.BLOOM_TABLE) + .where(rowIdCondition) + .asField(); + + final Field fe100kfp001 = DSL + .function( + "bloommatch", Boolean.class, smallColumn, + BLOOMDB.FILTER_EXPECTED_100000_FPP_001.FILTER + ); + final Field fe1000kfpp003 = DSL + .function( + "bloommatch", Boolean.class, mediumColumn, + BLOOMDB.FILTER_EXPECTED_1000000_FPP_003.FILTER + ); + final Field fe2500kfpp005 = DSL + .function( + "bloommatch", Boolean.class, largeColumn, + BLOOMDB.FILTER_EXPECTED_2500000_FPP_005.FILTER + ); + + final Condition noBloomFilter = BLOOMDB.FILTER_EXPECTED_100000_FPP_001.FILTER + .isNull() + .and( + BLOOMDB.FILTER_EXPECTED_1000000_FPP_003.FILTER + .isNull() + .and(BLOOMDB.FILTER_EXPECTED_2500000_FPP_005.FILTER.isNull()) + ); + final Condition queryCondition = fe100kfp001 + .eq(true) + .or(fe1000kfpp003.eq(true).or(fe2500kfpp005.eq(true).or(noBloomFilter))); + LOGGER.trace("ConditionWalker.emitElement bloomCondition part <{}>", queryCondition); + + return queryCondition; + } + + /** + * @param object object compared against + * @return true if object is same class and all object values are equal (tokenizer values are expected to point to same reference) + */ + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final IndexStatementCondition cast = (IndexStatementCondition) object; + return this.value.equals(cast.value) && + this.config.equals(cast.config) && + this.tokenizer == cast.tokenizer; // expects same reference + } +} diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/LatestCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/LatestCondition.java new file mode 100644 index 00000000..5422e00e --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/LatestCondition.java @@ -0,0 +1,90 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; + +import java.sql.Date; +import java.time.Instant; + +import static com.teragrep.pth_06.jooq.generated.journaldb.Journaldb.JOURNALDB; + +public final class LatestCondition implements QueryCondition { + private final String value; + + public LatestCondition(String value) { + this.value = value; + } + + public Condition condition() { + // SQL connection uses localTime in the session, so we use unix to come over the conversions + final Instant instant = Instant.ofEpochSecond(Integer.parseInt(value)); + final java.sql.Date timeQualifier = new Date(instant.toEpochMilli()); + Condition condition; + condition = JOURNALDB.LOGFILE.LOGDATE.lessOrEqual(timeQualifier); + condition = condition.and("UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H'))" + + " <= " + instant.getEpochSecond()); + // raw SQL used here since following not supported for mariadb: + // queryCondition = queryCondition.and(toTimestamp( + // regexpReplaceAll(JOURNALDB.LOGFILE.PATH, "((^.*\\/.*-)|(\\.log\\.gz.*))", ""), + // "YYYYMMDDHH24").lessOrEqual(Timestamp.from(instant))); + // to match + // 2021/09-27/sc-99-99-14-244/messages/messages-2021092722.gz.4 + // 2018/04-29/sc-99-99-14-245/f17/f17.logGLOB-2018042900.log.gz + // NOTE uses literal path + return condition; + } + + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final LatestCondition cast = (LatestCondition) object; + return this.value.equals(cast.value); + } +} \ No newline at end of file diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/QueryCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/QueryCondition.java new file mode 100644 index 00000000..01894dc8 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/QueryCondition.java @@ -0,0 +1,53 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; + +public interface QueryCondition { + Condition condition(); +} diff --git a/src/main/java/com/teragrep/pth_06/planner/walker/conditions/SourceTypeCondition.java b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/SourceTypeCondition.java new file mode 100644 index 00000000..e8b461d1 --- /dev/null +++ b/src/main/java/com/teragrep/pth_06/planner/walker/conditions/SourceTypeCondition.java @@ -0,0 +1,93 @@ +/* + * This program handles user requests that require archive access. + * Copyright (C) 2024 Suomen Kanuuna Oy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * + * Additional permission under GNU Affero General Public License version 3 + * section 7 + * + * If you modify this Program, or any covered work, by linking or combining it + * with other code, such other code is not for that reason alone subject to any + * of the requirements of the GNU Affero GPL version 3 as long as this Program + * is the same Program as licensed from Suomen Kanuuna Oy without any additional + * modifications. + * + * Supplemented terms under GNU Affero General Public License version 3 + * section 7 + * + * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified + * versions must be marked as "Modified version of" The Program. + * + * Names of the licensors and authors may not be used for publicity purposes. + * + * No rights are granted for use of trade names, trademarks, or service marks + * which are in The Program if any. + * + * Licensee must indemnify licensors and authors for any liability that these + * contractual assumptions impose on licensors and authors. + * + * To the extent this program is licensed as part of the Commercial versions of + * Teragrep, the applicable Commercial License may apply to this file if you as + * a licensee so wish it. + */ + +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.pth_06.planner.StreamDBClient; +import org.jooq.Condition; + +import static com.teragrep.pth_06.jooq.generated.streamdb.Streamdb.STREAMDB; + +public final class SourceTypeCondition implements QueryCondition { + private final String value; + private final String operation; + private final boolean streamQuery; + + public SourceTypeCondition(String value, String operation, boolean streamQuery) { + this.value = value; + this.operation = operation; + this.streamQuery = streamQuery; + } + + public Condition condition() { + Condition condition; + if (streamQuery) { + condition = STREAMDB.STREAM.STREAM_.like( + value.replace('*', '%') + ); + } else { + condition = + StreamDBClient.GetArchivedObjectsFilterTable.stream.like( + value.replace('*', '%').toLowerCase() + ); + } + if (operation.equalsIgnoreCase("NOT_EQUALS")) { + condition = condition.not(); + } + return condition; + } + + @Override + public boolean equals(final Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + final SourceTypeCondition cast = (SourceTypeCondition) object; + return this.streamQuery == cast.streamQuery && + this.value.equals(cast.value) && + this.operation.equals(cast.operation); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/EarliestConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/EarliestConditionTest.java new file mode 100644 index 00000000..5e3bb0a6 --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/EarliestConditionTest.java @@ -0,0 +1,36 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class EarliestConditionTest { + + @Test + void conditionTest() { + String e = "(\n" + + " \"journaldb\".\"logfile\".\"logdate\" >= date '1970-01-01'\n" + + " and (UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H')) >= 0)\n" + + ")"; + Condition elementCondition = new EarliestCondition("1000").condition(); + Assertions.assertEquals(e, elementCondition.toString()); + } + + @Test + void equalsTest() { + EarliestCondition eq1 = new EarliestCondition("946677600"); + eq1.condition(); + EarliestCondition eq2 = new EarliestCondition("946677600"); + Assertions.assertEquals(eq1, eq2); + Assertions.assertEquals(eq2, eq1); + } + + @Test + void notEqualsTest() { + EarliestCondition eq1 = new EarliestCondition("946677600"); + EarliestCondition notEq = new EarliestCondition("1000"); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, null); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/ElementConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/ElementConditionTest.java new file mode 100644 index 00000000..41996b7e --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/ElementConditionTest.java @@ -0,0 +1,177 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.pth_06.config.ConditionConfig; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.exception.SQLDialectNotSupportedException; +import org.jooq.impl.DSL; +import org.jooq.tools.jdbc.MockConnection; +import org.jooq.tools.jdbc.MockResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import javax.xml.parsers.DocumentBuilderFactory; + +class ElementConditionTest { + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + final Document document = Assertions.assertDoesNotThrow(() -> factory.newDocumentBuilder().newDocument()); + final DSLContext mockCtx = DSL.using(new MockConnection(ctx -> new MockResult[0])); + final ConditionConfig config = new ConditionConfig(mockCtx, false, true, false); + final ConditionConfig streamConfig = new ConditionConfig(mockCtx, true); + + @Test + void testStreamTags() { + String[] streamTags = {"index", "host", "sourcetype"}; + int loops = 0; + for (String tag : streamTags) { + Element element = document.createElement(tag); + element.setAttribute("value", "1000"); + element.setAttribute("operation", "EQUALS"); + Condition condition = new ElementCondition(element, streamConfig).condition(); + Assertions.assertTrue(condition.toString().contains("1000")); + loops++; + } + Assertions.assertEquals(loops, streamTags.length); + } + + @Test + void testIndexStatement() { + Element element = document.createElement("indexstatement"); + element.setAttribute("value", "searchTerm"); + element.setAttribute("operation", "EQUALS"); + Element element2 = document.createElement("indexstatement"); + element2.setAttribute("value", "searchTerm"); + element2.setAttribute("operation", "NOT_EQUALS"); + Assertions.assertThrows(SQLDialectNotSupportedException.class, new ElementCondition(element, config)::condition); + Assertions.assertThrows(IllegalStateException.class, new ElementCondition(element, streamConfig)::condition); + Assertions.assertThrows(IllegalStateException.class, new ElementCondition(element2, config)::condition); + } + + @Test + void testProvidedElementMissingValue() { + Element element = document.createElement("test"); + element.setAttribute("operation", "EQUALS"); + ElementCondition elementCondition = new ElementCondition(element, config); + ElementCondition streamElementCondition = new ElementCondition(element, streamConfig); + Assertions.assertThrows(IllegalStateException.class, elementCondition::condition); + Assertions.assertThrows(IllegalStateException.class, streamElementCondition::condition); + } + + @Test + void testProvidedElementMissingOperation() { + Element element = document.createElement("test"); + element.setAttribute("value", "1000"); + ElementCondition elementCondition = new ElementCondition(element, config); + ElementCondition streamElementCondition = new ElementCondition(element, streamConfig); + Assertions.assertThrows(IllegalStateException.class, elementCondition::condition); + Assertions.assertThrows(IllegalStateException.class, streamElementCondition::condition); + } + + @Test + void testTimeQualifiers() { + String[] tags = {"earliest", "latest", "index_earliest", "index_latest"}; + int loops = 0; + for (String tag : tags) { + Element element = document.createElement(tag); + element.setAttribute("value", "1000"); + element.setAttribute("operation", "EQUALS"); + Condition condition = new ElementCondition(element, config).condition(); + Assertions.assertTrue(condition.toString().contains("date")); + loops++; + } + Assertions.assertEquals(4, loops); + } + + @Test + void testInvalidStreamTags() { + String[] tags = {"earliest", "latest", "index_earliest", "index_latest", "indexstatement"}; + int loops = 0; + for (String tag : tags) { + Element element = document.createElement(tag); + element.setAttribute("value", "1000"); + element.setAttribute("operation", "EQUALS"); + Assertions.assertThrows(IllegalStateException.class, + () -> new ElementCondition(element, streamConfig).condition() + ); + loops++; + } + Assertions.assertEquals(5, loops); + } + + @Test + void invalidElementNameTest() { + Element element = document.createElement("test"); + element.setAttribute("value", "1000"); + element.setAttribute("operation", "EQUALS"); + Assertions.assertThrows(IllegalStateException.class, + () -> new ElementCondition(element, streamConfig).condition() + ); + Element element2 = document.createElement("hostindex"); + element2.setAttribute("value", "test"); + element2.setAttribute("operation", "EQUALS"); + Assertions.assertThrows(IllegalStateException.class, + () -> new ElementCondition(element2, streamConfig).condition() + ); + } + + @Test + void equalsTest() { + Element element = document.createElement("index"); + element.setAttribute("value", "f17"); + element.setAttribute("operation", "EQUALS"); + ElementCondition eq1 = new ElementCondition(element, config); + eq1.condition(); + ElementCondition eq2 = new ElementCondition(element, config); + Assertions.assertEquals(eq1, eq2); + } + + @Test + void notEqualsTest() { + Element element = document.createElement("index"); + element.setAttribute("value", "f17"); + element.setAttribute("operation", "EQUALS"); + Element anotherElement = document.createElement("index"); + anotherElement.setAttribute("value", "f11"); + anotherElement.setAttribute("operation", "EQUALS"); + ElementCondition eq1 = new ElementCondition(element, config); + ElementCondition notEq = new ElementCondition(anotherElement, config); + ElementCondition notEq2 = new ElementCondition(element, streamConfig); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, notEq2); + Assertions.assertNotEquals(notEq, notEq2); + Assertions.assertNotEquals(eq1, null); + } + + @Test + void notEqualsDifferentConfigTest() { + Element element = document.createElement("index"); + element.setAttribute("value", "f17"); + element.setAttribute("operation", "EQUALS"); + MockConnection conn = new MockConnection(ctx -> new MockResult[0]); + DSLContext ctx = DSL.using(conn); + ConditionConfig cfg1 = new ConditionConfig(ctx, false); + ConditionConfig cfg2 = new ConditionConfig(ctx, true); + ElementCondition eq = new ElementCondition(element, cfg1); + ElementCondition notEq = new ElementCondition(element, cfg2); + Assertions.assertNotEquals(eq, notEq); + } + + @Test + void notEqualsDifferentConnectionTest() { + Element element = document.createElement("index"); + element.setAttribute("value", "f17"); + element.setAttribute("operation", "EQUALS"); + MockConnection conn = new MockConnection(ctx -> new MockResult[0]); + MockConnection conn2 = new MockConnection(ctx -> new MockResult[0]); + DSLContext ctx = DSL.using(conn); + DSLContext ctx2 = DSL.using(conn2); + ConditionConfig cfg1 = new ConditionConfig(ctx, false); + ConditionConfig cfg2 = new ConditionConfig(ctx2, false); + ElementCondition eq = new ElementCondition(element, cfg1); + ElementCondition notEq = new ElementCondition(element, cfg2); + Assertions.assertNotEquals(eq, notEq); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/HostConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/HostConditionTest.java new file mode 100644 index 00000000..74599d94 --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/HostConditionTest.java @@ -0,0 +1,57 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class HostConditionTest { + + @Test + void conditionTest() { + HostCondition elementCondition = new HostCondition("f17", "EQUALS", false); + HostCondition streamElementCondition = new HostCondition("f17", "EQUALS", true); + String e = "\"getArchivedObjects_filter_table\".\"host\" like 'f17'"; + String eStream = "\"streamdb\".\"host\".\"name\" like 'f17'"; + Condition elementResult = elementCondition.condition(); + Condition streamElementResult = streamElementCondition.condition(); + Assertions.assertEquals(e, elementResult.toString()); + Assertions.assertEquals(eStream, streamElementResult.toString()); + } + + @Test + void negationTest() { + HostCondition elementCondition = new HostCondition("f17", "NOT_EQUALS", false); + HostCondition streamElementCondition = new HostCondition("f17", "NOT_EQUALS", true); + String e = "not (\"getArchivedObjects_filter_table\".\"host\" like 'f17')"; + String eStream = "not (\"streamdb\".\"host\".\"name\" like 'f17')"; + Condition elementResult = elementCondition.condition(); + Condition streamElementResult = streamElementCondition.condition(); + Assertions.assertEquals(e, elementResult.toString()); + Assertions.assertEquals(eStream, streamElementResult.toString()); + } + + @Test + void equalsTest() { + HostCondition eq1 = new HostCondition("946677600", "EQUALS", false); + eq1.condition(); + HostCondition eq2 = new HostCondition("946677600", "EQUALS", false); + HostCondition eq3 = new HostCondition("946677600", "EQUALS", true); + eq3.condition(); + HostCondition eq4 = new HostCondition("946677600", "EQUALS", true); + Assertions.assertEquals(eq1, eq2); + Assertions.assertEquals(eq2, eq1); + Assertions.assertEquals(eq3, eq4); + } + + @Test + void notEqualsTest() { + HostCondition eq1 = new HostCondition("946677600", "EQUALS", false); + HostCondition notEq = new HostCondition("1000", "EQUALS", false); + HostCondition notEq2 = new HostCondition("946677600", "EQUALS", true); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, null); + Assertions.assertNotEquals(eq1, notEq2); + Assertions.assertNotEquals(notEq, notEq2); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/IndexConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/IndexConditionTest.java new file mode 100644 index 00000000..644da64d --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/IndexConditionTest.java @@ -0,0 +1,52 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IndexConditionTest { + + @Test + void conditionTest() { + String e = "\"getArchivedObjects_filter_table\".\"directory\" like 'f17'"; + String eStream = "\"streamdb\".\"stream\".\"directory\" like 'f17'"; + Condition elementCondition = new IndexCondition("f17", "EQUALS", false).condition(); + Condition streamElementCondition = new IndexCondition("f17", "EQUALS", true).condition(); + Assertions.assertEquals(e, elementCondition.toString()); + Assertions.assertEquals(eStream, streamElementCondition.toString()); + } + + @Test + void negationTest() { + String e = "not (\"getArchivedObjects_filter_table\".\"directory\" like 'f17')"; + String eStream = "not (\"streamdb\".\"stream\".\"directory\" like 'f17')"; + Condition elementCondition = new IndexCondition("f17", "NOT_EQUALS", false).condition(); + Condition streamElementCondition = new IndexCondition("f17", "NOT_EQUALS", true).condition(); + Assertions.assertEquals(e, elementCondition.toString()); + Assertions.assertEquals(eStream, streamElementCondition.toString()); + } + + @Test + void equalsTest() { + IndexCondition eq1 = new IndexCondition("946677600", "EQUALS", false); + eq1.condition(); + IndexCondition eq2 = new IndexCondition("946677600", "EQUALS", false); + IndexCondition eq3 = new IndexCondition("946677600", "EQUALS", true); + eq3.condition(); + IndexCondition eq4 = new IndexCondition("946677600", "EQUALS", true); + Assertions.assertEquals(eq1, eq2); + Assertions.assertEquals(eq3, eq4); + } + + @Test + void notEqualsTest() { + IndexCondition eq1 = new IndexCondition("946677600", "EQUALS", false); + IndexCondition notEq = new IndexCondition("1000", "EQUALS", false); + IndexCondition notEq2 = new IndexCondition("946677600", "EQUALS", true); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, notEq2); + Assertions.assertNotEquals(notEq, notEq2); + Assertions.assertNotEquals(eq1, null); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/IndexStatementConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/IndexStatementConditionTest.java new file mode 100644 index 00000000..257a72a1 --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/IndexStatementConditionTest.java @@ -0,0 +1,48 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import com.teragrep.blf_01.Tokenizer; +import com.teragrep.pth_06.config.ConditionConfig; +import org.jooq.DSLContext; +import org.jooq.exception.SQLDialectNotSupportedException; +import org.jooq.impl.DSL; +import org.jooq.tools.jdbc.MockConnection; +import org.jooq.tools.jdbc.MockResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/** + * Requires database setup for full test + */ +public class IndexStatementConditionTest { + final ConditionConfig config = new ConditionConfig(DSL.using(new MockConnection(ctx -> new MockResult[0])), false, true, false); + final Tokenizer tokenizer = new Tokenizer(32); + @Test + void conditionTest() { + DSLContext ctx = DSL.using(new MockConnection(context -> new MockResult[0])); + // only tests that database access is tried as expected + Assertions.assertThrows(SQLDialectNotSupportedException.class, () -> + new IndexStatementCondition( + "value", + new ConditionConfig(ctx, false, true, false), + tokenizer + ).condition()); + } + + @Test + void equalsTest() { + IndexStatementCondition eq1 = new IndexStatementCondition("946677600", config, tokenizer); + IndexStatementCondition eq2 = new IndexStatementCondition("946677600", config, tokenizer); + Assertions.assertEquals(eq1, eq2); + Assertions.assertEquals(eq2, eq1); + } + + @Test + void notEqualsTest() { + IndexStatementCondition eq1 = new IndexStatementCondition("946677600", config, tokenizer); + IndexStatementCondition notEq = new IndexStatementCondition("1000", config, tokenizer); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, null); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/LatestConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/LatestConditionTest.java new file mode 100644 index 00000000..165d159f --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/LatestConditionTest.java @@ -0,0 +1,53 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class LatestConditionTest { + + @Test + void conditionTest() { + String e = "(\n" + + " \"journaldb\".\"logfile\".\"logdate\" <= date '1970-01-01'\n" + + " and (UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H')) <= 1000)\n" + + ")"; + Condition elementCondition = new LatestCondition("1000").condition(); + Assertions.assertEquals(e, elementCondition.toString()); + } + + @Test + void conditionUpdatedTest() { + String e = "(\n" + + " \"journaldb\".\"logfile\".\"logdate\" <= date '2000-01-01'\n" + + " and (UNIX_TIMESTAMP(STR_TO_DATE(SUBSTRING(REGEXP_SUBSTR(path,'[0-9]+(\\.log)?\\.gz(\\.[0-9]*)?$'), 1, 10), '%Y%m%d%H')) <= 946677600)\n" + + ")"; + Condition elementCondition = new LatestCondition("946677600").condition(); + Assertions.assertEquals(e, elementCondition.toString()); + } + + @Test + void equalsTest() { + IndexCondition eq1 = new IndexCondition("946677600", "EQUALS", false); + eq1.condition(); + IndexCondition eq2 = new IndexCondition("946677600", "EQUALS", false); + IndexCondition eq3 = new IndexCondition("946677600", "EQUALS", true); + eq3.condition(); + IndexCondition eq4 = new IndexCondition("946677600", "EQUALS", true); + Assertions.assertEquals(eq1, eq2); + Assertions.assertEquals(eq2, eq1); + Assertions.assertEquals(eq3, eq4); + } + + @Test + void notEqualsTest() { + IndexCondition eq1 = new IndexCondition("946677600", "EQUALS", false); + IndexCondition notEq = new IndexCondition("1000", "EQUALS", false); + IndexCondition notEq2 = new IndexCondition("946677600", "EQUALS", true); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, null); + Assertions.assertNotEquals(eq1, notEq2); + Assertions.assertNotEquals(notEq, notEq2); + } +} diff --git a/src/test/java/com/teragrep/pth_06/planner/walker/conditions/SourceTypeConditionTest.java b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/SourceTypeConditionTest.java new file mode 100644 index 00000000..145c3f8a --- /dev/null +++ b/src/test/java/com/teragrep/pth_06/planner/walker/conditions/SourceTypeConditionTest.java @@ -0,0 +1,53 @@ +package com.teragrep.pth_06.planner.walker.conditions; + +import org.jooq.Condition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class SourceTypeConditionTest { + + @Test + void conditionTest() { + String e = "\"getArchivedObjects_filter_table\".\"stream\" like 'f17'"; + String eStream = "\"streamdb\".\"stream\".\"stream\" like 'f17'"; + Condition elementCondition = new SourceTypeCondition("f17", "EQUALS", false).condition(); + Condition streamElementCondition = new SourceTypeCondition("f17", "EQUALS", true).condition(); + Assertions.assertEquals(e, elementCondition.toString()); + Assertions.assertEquals(eStream, streamElementCondition.toString()); + } + + @Test + void negationTest() { + String e = "not (\"getArchivedObjects_filter_table\".\"stream\" like 'f17')"; + String eStream = "not (\"streamdb\".\"stream\".\"stream\" like 'f17')"; + Condition elementCondition = new SourceTypeCondition("f17", "NOT_EQUALS", false).condition(); + Condition streamElementCondition = new SourceTypeCondition("f17", "NOT_EQUALS", true).condition(); + Assertions.assertEquals(e, elementCondition.toString()); + Assertions.assertEquals(eStream, streamElementCondition.toString()); + } + + @Test + void equalsTest() { + IndexCondition eq1 = new IndexCondition("946677600", "EQUALS", false); + eq1.condition(); + IndexCondition eq2 = new IndexCondition("946677600", "EQUALS", false); + IndexCondition eq3 = new IndexCondition("946677600", "EQUALS", true); + eq3.condition(); + IndexCondition eq4 = new IndexCondition("946677600", "EQUALS", true); + Assertions.assertEquals(eq1, eq2); + Assertions.assertEquals(eq2, eq1); + Assertions.assertEquals(eq3, eq4); + } + + @Test + void notEqualsTest() { + IndexCondition eq1 = new IndexCondition("946677600", "EQUALS", false); + IndexCondition notEq = new IndexCondition("1000", "EQUALS", false); + IndexCondition notEq2 = new IndexCondition("1000", "EQUALS", true); + Assertions.assertNotEquals(eq1, notEq); + Assertions.assertNotEquals(notEq, eq1); + Assertions.assertNotEquals(eq1, null); + Assertions.assertNotEquals(eq1, notEq2); + Assertions.assertNotEquals(notEq, notEq2); + } +}