From 054a11454312b6db7162878fbdaf54697f70e773 Mon Sep 17 00:00:00 2001 From: Raphael Pinto Date: Fri, 4 Aug 2023 17:52:10 -0300 Subject: [PATCH 1/4] fix: refactory rulesheet expiration empty response --- api.http | 2 +- services/rulles.go | 104 +++++++++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/api.http b/api.http index 59b93ef..21457d2 100644 --- a/api.http +++ b/api.http @@ -1,4 +1,4 @@ -POST http://localhost:8000/api/v1/eval/jamie-menu-64623317-d00c-4508-8db3-613d1ed24638/ +POST http://localhost:8000/api/v1/eval/mobilepf/latest Content-Type: application/json { diff --git a/services/rulles.go b/services/rulles.go index e7475ae..04c925f 100644 --- a/services/rulles.go +++ b/services/rulles.go @@ -3,6 +3,7 @@ package services import ( "bytes" "fmt" + "strconv" "strings" "sync" "text/template" @@ -47,19 +48,6 @@ type knowledgeBaseInfo struct { Version string } -// The type `knowledgeBaseCache` contains a knowledge base and its expiration date. -// @property KnowledgeBase - KnowledgeBase is a pointer to an ast.KnowledgeBase struct, which likely -// contains information or data related to a specific domain or topic. This struct may include rules, -// facts, and other information that can be used for reasoning or decision-making in a particular -// context. -// @property ExpirationDate - ExpirationDate is a property of the knowledgeBaseCache struct that -// represents the date and time when the cached knowledge base will expire and need to be refreshed. -// This is useful for ensuring that the cached data is not used indefinitely and remains up-to-date. -type knowledgeBaseCache struct { - KnowledgeBase *ast.KnowledgeBase - ExpirationDate time.Time -} - // LoadRemoteGRL function is responsible for loading GRL (Grule Rule Language) rules from a remote location, such as a GitLab repository, // and constructing a rule from them using the builder.NewRuleBuilder function. It takes the knowledge base name (rulesheet) and the // knowledge base version as parameters. @@ -101,6 +89,8 @@ func (s Eval) LoadRemoteGRL(knowledgeBaseName string, version string) error { // called by multiple goroutines. By using a mutex, it ensures that only one goroutine can execute the Eval method at a time, preventing race conditions and ensuring proper execution of the method. var evalMutex sync.Mutex +var getMutex sync.Mutex + // IEval interface defines methods for loading and evaluating knowledge bases in Go. // // Property @@ -120,27 +110,24 @@ type IEval interface { // EvalService is a variable type of `IEval` and initializing it with a new instance of the `Eval` struct created by calling the `NewEval()` // function. This variable can be used to access the methods defined in the `IEval` interface. -var loadMutex sync.Mutex - -// EvalService ... -var EvalService IEval = NewEval() +var EvalService IEval = NewEval(config.GetConfig()) // Eval type contains a reference to a knowledge library in Go's abstract syntax tree. // // Property: // - knowledgeLibrary - `knowledgeLibrary` is a pointer to an `ast.KnowledgeLibrary` object. Itis a property of the `Eval` struct. type Eval struct { - knowledgeLibrary *ast.KnowledgeLibrary - knowledgeBaseCache map[knowledgeBaseInfo]*knowledgeBaseCache - versionTTL int64 + knowledgeLibrary *ast.KnowledgeLibrary + expirationMap map[knowledgeBaseInfo]time.Time + versionTTL int64 } // NewEval creates a new instance of the Eval struct with an empty knowledge library. -func NewEval() Eval { +func NewEval(config *config.Config) Eval { return Eval{ - knowledgeLibrary: ast.NewKnowledgeLibrary(), - knowledgeBaseCache: map[knowledgeBaseInfo]*knowledgeBaseCache{}, - versionTTL: config.GetConfig().KnowledgeBaseVersionTTL, + knowledgeLibrary: ast.NewKnowledgeLibrary(), + expirationMap: map[knowledgeBaseInfo]time.Time{}, + versionTTL: config.KnowledgeBaseVersionTTL, } } @@ -169,54 +156,54 @@ func (s Eval) GetDefaultKnowledgeBase() *ast.KnowledgeBase { // the cache and returns it if it does. If it does not exist or has expired, it loads the knowledge // base from a remote source using the `LoadRemoteGRL func (s Eval) GetKnowledgeBase(knowledgeBaseName string, version string) (*ast.KnowledgeBase, *errors.RequestError) { - info := knowledgeBaseInfo{KnowledgeBaseName: knowledgeBaseName, Version: version} - existing := s.knowledgeBaseCache[info] - if existing == nil { - var ExpirationDate = time.Now().Add(time.Duration(s.versionTTL) * time.Second) + getMutex.Lock() + defer getMutex.Unlock() - existing = &knowledgeBaseCache{ - KnowledgeBase: s.GetKnowledgeLibrary().GetKnowledgeBase(knowledgeBaseName, version), - ExpirationDate: ExpirationDate, - } + info := knowledgeBaseInfo{KnowledgeBaseName: knowledgeBaseName, Version: version} - s.knowledgeBaseCache[info] = existing + base := s.GetKnowledgeLibrary().GetKnowledgeBase(knowledgeBaseName, version) - return existing.KnowledgeBase, nil - } - if existing.KnowledgeBase.Version != "latest" && len(existing.KnowledgeBase.RuleEntries) > 0 { - return existing.KnowledgeBase, nil - } + expirable := true - if existing.ExpirationDate.After(time.Now()) && len(existing.KnowledgeBase.RuleEntries) > 0 { - return existing.KnowledgeBase, nil + // If the version is a number, it isn't expirable + if _, err := strconv.Atoi(version); err == nil { + expirable = false } - loadMutex.Lock() + expired := expirable && s.isKnowledgeBaseVersionExpired(info) - for key := range existing.KnowledgeBase.RuleEntries { + // If the version isn't expired and there are rules, we must retrieve the version + if !expired && len(base.RuleEntries) > 0 { + return base, nil + } - s.knowledgeLibrary.RemoveRuleEntry(key, knowledgeBaseName, version) + // If the version is expired, we must invalidate its rules + if expired { + log.Trace("GetKnowledgeBase: [expired]") + for key := range base.RuleEntries { + s.knowledgeLibrary.RemoveRuleEntry(key, knowledgeBaseName, version) + } } err := s.LoadRemoteGRL(knowledgeBaseName, version) + if err != nil { log.Errorf("Erro on load: %v", err) - loadMutex.Unlock() return nil, &errors.RequestError{Message: "Error on load KnowledgeBase and/or version", StatusCode: 500} } - if !(len(existing.KnowledgeBase.RuleEntries) > 0) { - loadMutex.Unlock() + base = s.GetKnowledgeLibrary().GetKnowledgeBase(knowledgeBaseName, version) + + if len(base.RuleEntries) == 0 { return nil, &errors.RequestError{Message: "KnowledgeBase or version not found", StatusCode: 404} } - loadMutex.Unlock() - - existing.KnowledgeBase = s.GetKnowledgeLibrary().GetKnowledgeBase(knowledgeBaseName, version) - existing.ExpirationDate = time.Now().Add(time.Duration(s.versionTTL) * time.Second) + if expirable { + s.expirationMap[info] = time.Now().Add(time.Duration(s.versionTTL) * time.Second) + } - return existing.KnowledgeBase, nil + return base, nil } @@ -276,3 +263,18 @@ func (s Eval) Eval(ctx *types.Context, knowledgeBase *ast.KnowledgeBase) (result return } + +func (s Eval) isKnowledgeBaseVersionExpired(info knowledgeBaseInfo) bool { + + expireDate, ok := s.expirationMap[info] + + if !ok { + return false + } + + if expireDate.After(time.Now()) { + return false + } + + return true +} From c7935f635b0d90820950a029364452fa6ac083b9 Mon Sep 17 00:00:00 2001 From: Raphael Pinto Date: Fri, 4 Aug 2023 17:52:27 -0300 Subject: [PATCH 2/4] docs: add ttl env sample --- .env.sample | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 4d91d06..da76e16 100644 --- a/.env.sample +++ b/.env.sample @@ -2,4 +2,6 @@ FEATWS_RULLER_PORT=8000 FEATWS_RULLER_RESOURCE_LOADER_URL=https://myrepo.com/{knowledgeBase}/{version}/rules.grl FEATWS_RULLER_RESOURCE_LOADER_HEADERS=PRIVATE-TOKEN:123123d12d12d1 TELEMETRY_EXPORTER_URL=http://jaeger:YOURPORT -FEATWS_RULLER_API_KEY=abc123 \ No newline at end of file +FEATWS_RULLER_API_KEY=abc123 + +FEATWS_RULLER_KNOWLEDGE_BASE_VERSION_TTL=5 From 7fc5ddd9a8cce77d0700ffd3912973ae9761e45e Mon Sep 17 00:00:00 2001 From: Raphael Pinto Date: Fri, 4 Aug 2023 18:44:16 -0300 Subject: [PATCH 3/4] docs: add other example request --- .gitignore | 1 + api.http | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 05bd204..1303379 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ example.json *.env *.out vendor +test.json diff --git a/api.http b/api.http index 21457d2..60eaed2 100644 --- a/api.http +++ b/api.http @@ -1,4 +1,5 @@ -POST http://localhost:8000/api/v1/eval/mobilepf/latest +#POST http://localhost:8000/api/v1/eval/mobilepf/latest +POST http://localhost:8000/api/v1/eval/jamie-menu-64623317-d00c-4508-8db3-613d1ed24638/ Content-Type: application/json { From 5916a81e1625d002e276101ce5c933e2373726c9 Mon Sep 17 00:00:00 2001 From: Raphael Pinto Date: Fri, 4 Aug 2023 18:49:29 -0300 Subject: [PATCH 4/4] docs: adjusted --- services/rulles.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/rulles.go b/services/rulles.go index 04c925f..e42eafd 100644 --- a/services/rulles.go +++ b/services/rulles.go @@ -149,12 +149,13 @@ func (s Eval) GetDefaultKnowledgeBase() *ast.KnowledgeBase { return s.GetKnowledgeLibrary().GetKnowledgeBase(DefaultKnowledgeBaseName, DefaultKnowledgeBaseVersion) } -// GetKnowledgeBase is a method in the `Eval` struct that retrieves a knowledge base from a cache or -// loads it from a remote source if it is not found in the cache. It takes in the name and version of +// GetKnowledgeBase is a method in the `Eval` struct that retrieves a knowledge base handling a possible +// expiration in the rulesheet if it reach the expiration date or loads it from a remote source if it is +// not found in the cache. It takes in the name and version of // the knowledge base as parameters and returns a pointer to the `ast.KnowledgeBase` struct and a -// `*errors.RequestError` if there is an error. The method first checks if the knowledge base exists in -// the cache and returns it if it does. If it does not exist or has expired, it loads the knowledge -// base from a remote source using the `LoadRemoteGRL +// `*errors.RequestError` if there is an error. The method first checks if the knowledge base is expired. +// If it does not exist or has expired, it loads the knowledge +// base from a remote source using the `LoadRemoteGRL` func (s Eval) GetKnowledgeBase(knowledgeBaseName string, version string) (*ast.KnowledgeBase, *errors.RequestError) { getMutex.Lock()