Skip to content

Commit

Permalink
Use more direct access to variables in boolean evaluator
Browse files Browse the repository at this point in the history 
Accessing variables through maps leads to a lot of string operations.
Exprtk checks variable names for validity each time. This commit
uses vectors to pass matching results and pointers to set variables
for exprtk evaluator.
  • Loading branch information
@variar
variar committed Jun 2, 2021
1 parent d73edf1 commit 24cc11b
Show file tree
Hide file tree
Showing 5 changed files with 55 additions and 54 deletions.
7 changes: 5 additions & 2 deletions src/logdata/include/data/booleanevaluator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/

#include <exprtk.hpp>
#include <robin_hood.h>

#include "regularexpressionpattern.h"

Expand All @@ -37,13 +36,17 @@ class BooleanExpressionEvaluator {
return errorString_;
}

bool evaluate( const robin_hood::unordered_flat_map<std::string, bool>& variables );
bool evaluate( const std::vector<unsigned char>& variables );

private:
bool isValid_ = true;
std::string errorString_;



exprtk::symbol_table<double> symbols_;
exprtk::expression<double> expression_;
exprtk::parser<double> parser_;

std::vector<double*> variables_;
};
40 changes: 19 additions & 21 deletions src/logdata/include/data/hsregularexpression.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,12 @@
#ifndef KLOGG_HS_REGULAR_EXPRESSION
#define KLOGG_HS_REGULAR_EXPRESSION

#include <qregularexpression.h>
#include <string_view>
#include <unordered_map>
#include <variant>
#include <vector>

#include <QRegularExpression>
#include <QString>
#include <vector>

#include <robin_hood.h>

#ifdef KLOGG_HAS_HS
#include <hs.h>
Expand All @@ -39,7 +35,7 @@

#include "regularexpressionpattern.h"

using MatchedPatterns = robin_hood::unordered_flat_map<std::string, bool>;
using MatchedPatterns = std::vector<unsigned char>;
using MatchingResult = std::variant<bool, MatchedPatterns>;

class DefaultRegularExpressionMatcher {
Expand All @@ -48,40 +44,42 @@ class DefaultRegularExpressionMatcher {
const std::vector<RegularExpressionPattern>& patterns )
{
for ( const auto& pattern : patterns ) {
regexp_.emplace( pattern.id(), static_cast<QRegularExpression>( pattern ) );
regexp_.push_back( static_cast<QRegularExpression>( pattern ) );
}
}

std::vector<std::string> match( const QString& data ) const
{
std::vector<std::string> matchingPatterns;
for ( const auto& regexp : regexp_ ) {
if ( regexp.second.match( data ).hasMatch() ) {
matchingPatterns.push_back( regexp.first );
}
}
return matchingPatterns;
}
// std::vector<std::string> match( const QString& data ) const
// {
// std::vector<std::string> matchingPatterns;
// for ( const auto& regexp : regexp_ ) {
// if ( regexp.second.match( data ).hasMatch() ) {
// matchingPatterns.push_back( regexp.first );
// }
// }
// return matchingPatterns;
// }

MatchingResult match( const std::string_view& utf8Data ) const
{
MatchedPatterns matchingPatterns;
for ( const auto& regexp : regexp_ ) {
const auto hasMatch = regexp.second
matchingPatterns.resize( regexp_.size() );
for ( auto index = 0u; index < regexp_.size(); ++index ) {
const auto hasMatch = regexp_[ index ]
.match( QString::fromUtf8(
utf8Data.data(), static_cast<int>( utf8Data.size() ) ) )
.hasMatch();
if ( regexp_.size() == 1 ) {
return hasMatch;
}

matchingPatterns.emplace( regexp.first, hasMatch );
matchingPatterns[ index ] = hasMatch;
}

return matchingPatterns;
}

private:
robin_hood::unordered_flat_map<std::string, QRegularExpression> regexp_;
std::vector<QRegularExpression> regexp_;
};

#ifdef KLOGG_HAS_HS
Expand Down
21 changes: 16 additions & 5 deletions src/logdata/src/booleanevaluator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,39 @@

#include "booleanevaluator.h"

#include "log.h"

BooleanExpressionEvaluator::BooleanExpressionEvaluator(
const std::string& expression, const std::vector<RegularExpressionPattern>& patterns )
{
variables_.reserve( patterns.size() );

for ( const auto& p : patterns ) {
symbols_.create_variable( p.id() );
if ( symbols_.create_variable( p.id() ) ) {
variables_.push_back( &symbols_.get_variable( p.id() )->ref() );
}
}

expression_.register_symbol_table( symbols_ );
isValid_ = parser_.compile( expression, expression_ );
if ( !isValid_ ) {
errorString_ = parser_.error();
}
}

bool BooleanExpressionEvaluator::evaluate(
const robin_hood::unordered_flat_map<std::string, bool>& variables )
bool BooleanExpressionEvaluator::evaluate( const std::vector<unsigned char>& variables )
{
if ( !isValid() ) {
return false;
}

for ( const auto& result : variables ) {
symbols_.get_variable( result.first )->ref() = result.second ? 1 : 0;
if ( variables_.size() != variables.size() ) {
LOG_ERROR << "Wrong number of matched patterns";
return false;
}

for ( auto index = 0u; index < variables_.size(); ++index ) {
*variables_[ index ] = variables[ index ];
}

return expression_.value() > 0;
Expand Down
36 changes: 14 additions & 22 deletions src/logdata/src/hsregularexpression.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@

#include <algorithm>
#include <iterator>
#include <qregularexpression.h>
#include <robin_hood.h>
#include <string_view>
#include <vector>

#include <QRegularExpression>

#ifdef KLOGG_HAS_HS
#include "hsregularexpression.h"

Expand All @@ -31,8 +32,8 @@
namespace {

struct MatcherContext {
const std::vector<std::string>& patternIds;
robin_hood::unordered_flat_set<std::string>& matchingPatterns;
const std::size_t patternsCount = 0;
MatchedPatterns& matchingPatterns;
bool hasMatch = false;
};

Expand All @@ -45,12 +46,12 @@ int matchCallback( unsigned int id, unsigned long long from, unsigned long long

MatcherContext* matchContext = static_cast<MatcherContext*>( context );

if ( matchContext->patternIds.size() == 1 ) {
if ( matchContext->patternsCount == 1 ) {
matchContext->hasMatch = true;
return 1;
}
else {
const auto& patternId = matchContext->patternIds[ id ];
matchContext->matchingPatterns.insert( patternId );
matchContext->matchingPatterns[ id ] = true;
}

return 0;
Expand All @@ -71,26 +72,17 @@ MatchingResult HsMatcher::match( const std::string_view& utf8Data ) const
if ( !scratch_ || !database_ ) {
return {};
}
robin_hood::unordered_flat_set<std::string> matchingPatterns;
MatcherContext context{ patternIds_, matchingPatterns };
MatchedPatterns matchingPatterns( patternIds_.size() );

MatcherContext context{ patternIds_.size(), matchingPatterns };
hs_scan( database_.get(), utf8Data.data(), static_cast<unsigned int>( utf8Data.size() ), 0,
scratch_.get(), matchCallback, static_cast<void*>( &context ) );

if ( patternIds_.size() > 1 ) {
robin_hood::unordered_map<std::string, bool> results;
for ( const auto& patternId : patternIds_ ) {
results[ patternId ] = false;
}

for ( const auto& match : matchingPatterns ) {
results[ match ] = true;
}

return results;
if ( context.patternsCount == 1 ) {
return context.hasMatch;
}
else {
return context.hasMatch;
return matchingPatterns;
}
}

Expand Down
5 changes: 1 addition & 4 deletions src/logdata/src/regularexpression.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,8 @@
#include <exception>
#include <memory>
#include <string>
#include <unordered_map>
#include <variant>

#include <robin_hood.h>

#include "configuration.h"
#include "log.h"
#include "overload_visitor.h"
Expand Down Expand Up @@ -179,7 +176,7 @@ bool PatternMatcher::hasMatchInternal( std::string_view line ) const
}
else {
MatchedPatterns matchedPatterns;
matchedPatterns.emplace( mainPatternId_, hasMatch );
matchedPatterns.push_back( hasMatch );
return evaluator_->evaluate( matchedPatterns );
}
},
Expand Down

0 comments on commit 24cc11b

Please sign in to comment.