Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

expect regex extracted tokens in database bloom filters #103

Merged
merged 26 commits into from
Nov 11, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
b9d2731
expect regex extracted tokens in database bloom filters
elliVM Oct 18, 2024
465cf83
add RegexExtractedValueTest
elliVM Oct 22, 2024
7d4559d
remove unnecessary test
elliVM Oct 22, 2024
945e839
TokenizedValue: call tokenizer only when needed, clean up tests
elliVM Oct 22, 2024
2a8d97c
clear up exception message in BloomFilterFromRecord
elliVM Oct 22, 2024
b3ac348
BloomFilterFromRecord: remove ULong.longValue() from constructor, cla…
elliVM Oct 22, 2024
0d72406
set logger level to debug when indexstatement is reached with bloom d…
elliVM Oct 22, 2024
a682789
remove consumer class and use for loop in TableFilters
elliVM Oct 22, 2024
b354614
use try with resources and add comments on equals methods about DSLCo…
elliVM Oct 22, 2024
5f907c8
add Tokenizable interface and decorators, rename BloomFilterFromRecor…
elliVM Oct 23, 2024
0bf7410
add missing assertion to test
elliVM Oct 23, 2024
31e0bf9
move method after constructors
elliVM Oct 23, 2024
f370356
fix hard coded filter size and fix testing that different sizes are a…
elliVM Oct 25, 2024
eda9878
refactor code to simplify, add testing for SQL temp table values crea…
elliVM Oct 28, 2024
aefb66a
use UncheckedIOException constructor
elliVM Oct 29, 2024
73efeb9
TableFilters returns a batch that CategoryTableWithFilters executes
elliVM Oct 29, 2024
441e40b
add test for SafeBatch
elliVM Oct 29, 2024
4844ebd
update comments and clean up code, add constructors for RegexLikeCond…
elliVM Oct 29, 2024
145fcb6
use qualified names update tests
elliVM Oct 29, 2024
40c2165
more descriptive naming of methods and variables, update comments, ja…
elliVM Oct 30, 2024
6513861
apply spotless
elliVM Oct 30, 2024
adc7faf
add missing hashCode() methods
elliVM Oct 31, 2024
2c01634
don't wrap jooq.Batch object and execute in CategoryTableWithFilters,…
elliVM Oct 31, 2024
34efb98
throw exception if search term filter tokens size larger than expecte…
elliVM Nov 1, 2024
7412ec2
improve TableFiltersTest and TokensAsStringsTest
elliVM Nov 1, 2024
7dea6f7
allow search term filter tokens to be larger than expected tokens
elliVM Nov 11, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,18 @@
<version>2.2.224</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<version>3.16.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.github.hakky54</groupId>
<artifactId>logcaptor</artifactId>
<version>2.9.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/teragrep/pth_06/planner/StreamDBClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -344,9 +344,9 @@ private Table<Record> getTableStatement(Condition journaldbCondition, Date day)
.on(JOURNALDB.LOGFILE.HOST_ID.eq(GetArchivedObjectsFilterTable.host_id).and(JOURNALDB.LOGFILE.LOGTAG.eq(GetArchivedObjectsFilterTable.tag)));

if (bloomEnabled) {
Set<Table<?>> tables = walker.patternMatchTables();
final Set<Table<?>> tables = walker.patternMatchTables();
if (!tables.isEmpty()) {
for (Table<?> table : tables) {
for (final Table<?> table : tables) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Left join pattern match table: <{}>", table.getName());
}
Expand Down
168 changes: 0 additions & 168 deletions src/main/java/com/teragrep/pth_06/planner/TableFilters.java

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,10 @@
* Teragrep, the applicable Commercial License may apply to this file if you as
* a licensee so wish it.
*/
package com.teragrep.pth_06.planner;

import com.teragrep.pth_06.planner.walker.conditions.QueryCondition;
package com.teragrep.pth_06.planner.bloomfilter;

public interface CategoryTable {

void create();

void insertFilters();

QueryCondition bloommatchCondition();
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,9 @@
* Teragrep, the applicable Commercial License may apply to this file if you as
* a licensee so wish it.
*/
package com.teragrep.pth_06.planner;
package com.teragrep.pth_06.planner.bloomfilter;

import com.teragrep.pth_06.config.ConditionConfig;
import com.teragrep.pth_06.planner.walker.conditions.CategoryTableCondition;
import com.teragrep.pth_06.planner.walker.conditions.QueryCondition;
import org.jooq.*;
import org.jooq.impl.DSL;
import org.slf4j.Logger;
Expand Down Expand Up @@ -88,40 +86,25 @@ public final class CategoryTableImpl implements CategoryTable {
private final DSLContext ctx;
private final Table<?> originTable;
private final long bloomTermId;
private final CategoryTableCondition tableCondition;
private final TableFilters tableFilters;

public CategoryTableImpl(ConditionConfig config, Table<?> originTable, String value) {
this(
config.context(),
originTable,
config.bloomTermId(),
new CategoryTableCondition(originTable, config.bloomTermId()),
new TableFilters(config.context(), originTable, config.bloomTermId(), value)
);
}

public CategoryTableImpl(DSLContext ctx, Table<?> originTable, long bloomTermId, String value) {
this(
ctx,
originTable,
bloomTermId,
new CategoryTableCondition(originTable, bloomTermId),
new TableFilters(ctx, originTable, bloomTermId, value)
);
this(ctx, originTable, bloomTermId, new TableFilters(ctx, originTable, bloomTermId, value));
}

public CategoryTableImpl(
DSLContext ctx,
Table<?> originTable,
long bloomTermId,
CategoryTableCondition tableCondition,
TableFilters tableFilters
) {
public CategoryTableImpl(DSLContext ctx, Table<?> originTable, long bloomTermId, TableFilters tableFilters) {
this.ctx = ctx;
this.originTable = originTable;
this.bloomTermId = bloomTermId;
this.tableCondition = tableCondition;
this.tableFilters = tableFilters;
}

Expand All @@ -144,19 +127,6 @@ public void create() {
indexStep.execute();
}

public void insertFilters() {
tableFilters.insertFiltersIntoCategoryTable();
}

/**
* Row condition that selects the same sized filter arrays from this category table and the origin table.
*
* @return condition
*/
public QueryCondition bloommatchCondition() {
return tableCondition;
}

/**
* Equal only if all object parameters are same value and the instances of DSLContext are same
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,34 +43,34 @@
* Teragrep, the applicable Commercial License may apply to this file if you as
* a licensee so wish it.
*/
package com.teragrep.pth_06.planner;
package com.teragrep.pth_06.planner.bloomfilter;

import com.teragrep.pth_06.planner.walker.conditions.QueryCondition;
import org.jooq.DSLContext;
import org.jooq.Table;

/**
* Decorator that inserts category tables filter types into database
* Decorator that inserts category tables filter types into the table
*/
public final class SearchTermFiltersInserted implements CategoryTable {
public final class CategoryTableWithFilters implements CategoryTable {

private final CategoryTable origin;
private final TableFilters filters;

public SearchTermFiltersInserted(final CategoryTable origin) {
public CategoryTableWithFilters(DSLContext ctx, Table<?> origin, long bloomTermId, String searchTerm) {
this(
new CategoryTableImpl(ctx, origin, bloomTermId, searchTerm),
new TableFilters(ctx, origin, bloomTermId, searchTerm)
);
}

public CategoryTableWithFilters(CategoryTable origin, TableFilters filters) {
this.origin = origin;
this.filters = filters;
}

@Override
public void create() {
origin.create();
}

@Override
public void insertFilters() {
origin.insertFilters();
}

@Override
public QueryCondition bloommatchCondition() {
insertFilters();
return origin.bloommatchCondition();
filters.insertFiltersIntoCategoryTable();
}
}
Loading