Skip to content

Commit

Permalink
DB Access Layer Merges: GetTablePattern ... (sonic-net#103)
Browse files Browse the repository at this point in the history 
Added the following APIs:

ExistKeysPattern()
GetTablePattern()
Key.IsAllKeyPattern()
Go UT, and Benchmark Tests for the same
TranslibDBScriptFail error type to translib/tlerr package
  • Loading branch information
@a-barboza
a-barboza authored Sep 26, 2023
1 parent 4cfc882 commit 42ca0a6
Show file tree
Hide file tree
Showing 6 changed files with 864 additions and 0 deletions.
9 changes: 9 additions & 0 deletions translib/db/db_key.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,15 @@ func (k Key) Matches(pattern Key) bool {
return true
}

// IsAllKeyPattern returns true if it is an all key wildcard pattern.
// (i.e. A key with a single component "*")
func (k *Key) IsAllKeyPattern() bool {
if (len(k.Comp) == 1) && (k.Comp[0] == "*") {
return true
}
return false
}

// patternMatch checks if the value matches a key pattern.
// vIndex and pIndex are start positions of value and pattern strings to match.
// Mimics redis pattern matcher - i.e, glob like pattern matcher which
Expand Down
200 changes: 200 additions & 0 deletions translib/db/db_keys_pattern.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
////////////////////////////////////////////////////////////////////////////////
// //
// Copyright 2022 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
// its subsidiaries. //
// //
// Licensed under the Apache License, Version 2.0 (the "License"); //
// you may not use this file except in compliance with the License. //
// You may obtain a copy of the License at //
// //
// http://www.apache.org/licenses/LICENSE-2.0 //
// //
// Unless required by applicable law or agreed to in writing, software //
// distributed under the License is distributed on an "AS IS" BASIS, //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and //
// limitations under the License. //
// //
////////////////////////////////////////////////////////////////////////////////

package db

import (
"time"

"github.com/Azure/sonic-mgmt-common/translib/tlerr"
"github.com/go-redis/redis/v7"
"github.com/golang/glog"
)

////////////////////////////////////////////////////////////////////////////////
// Exported Types //
////////////////////////////////////////////////////////////////////////////////

////////////////////////////////////////////////////////////////////////////////
// Exported Functions //
////////////////////////////////////////////////////////////////////////////////

// ExistKeysPattern checks if a key pattern exists in a table
// Note:
// 1. Statistics do not capture when the CAS Tx Cache results in a quicker
// response. This is to avoid mis-interpreting per-connection statistics.
func (d *DB) ExistKeysPattern(ts *TableSpec, pat Key) (bool, error) {

var err error
var exists bool

// ExistsKeysPatternHits
// Time Start
var cacheHit bool
var now time.Time
var dur time.Duration
var stats Stats

if (d == nil) || (d.client == nil) {
return exists, tlerr.TranslibDBConnectionReset{}
}

if d.dbStatsConfig.TimeStats {
now = time.Now()
}

if glog.V(3) {
glog.Info("ExistKeysPattern: Begin: ", "ts: ", ts, "pat: ", pat)
}

defer func() {
if err != nil {
glog.Error("ExistKeysPattern: ts: ", ts, " err: ", err)
}
if glog.V(3) {
glog.Info("ExistKeysPattern: End: ts: ", ts, " exists: ", exists)
}
}()

// If pseudoDB then follow the path of !IsWriteDisabled with Tx Cache. TBD.

if !d.Opts.IsWriteDisabled {

// If Write is enabled, then just call GetKeysPattern() and check
// for now.

// Optimization: Check the DBL CAS Tx Cache for added entries.
for k := range d.txTsEntryMap[ts.Name] {
key := d.redis2key(ts, k)
if key.Matches(pat) {
if len(d.txTsEntryMap[ts.Name][k].Field) > 0 {
exists = true
break
}
// Removed entries/fields, can't help much, since we'd have
// to compare against the DB retrieved keys anyway
}
}

if !exists {
var getKeys []Key
if getKeys, err = d.GetKeysPattern(ts, pat); (err == nil) && len(getKeys) > 0 {

exists = true
}
}

} else if d.dbCacheConfig.PerConnection &&
d.dbCacheConfig.isCacheTable(ts.Name) {

// Check PerConnection cache first, [Found = SUCCESS return]
var keys []Key
if table, ok := d.cache.Tables[ts.Name]; ok {
if keys, ok = table.patterns[d.key2redis(ts, pat)]; ok && len(keys) > 0 {

exists = true
cacheHit = true

}
}
}

// Run Lua script [Found = SUCCESS return]
if d.Opts.IsWriteDisabled && !exists {

//glog.Info("ExistKeysPattern: B4= ", luaScriptExistsKeysPatterns.Hash())

var luaExists interface{}
if luaExists, err = luaScriptExistsKeysPatterns.Run(d.client,
[]string{d.key2redis(ts, pat)}).Result(); err == nil {

if existsString, ok := luaExists.(string); !ok {
err = tlerr.TranslibDBScriptFail{
Description: "Unexpected response"}
} else if existsString == "true" {
exists = true
} else if existsString != "false" {
err = tlerr.TranslibDBScriptFail{Description: existsString}
}
}

//glog.Info("ExistKeysPattern: AF= ", luaScriptExistsKeysPatterns.Hash())
}

// Time End, Time, Peak
if d.dbStatsConfig.TableStats {
stats = d.stats.Tables[ts.Name]
} else {
stats = d.stats.AllTables
}

stats.Hits++
stats.ExistsKeyPatternHits++
if cacheHit {
stats.ExistsKeyPatternCacheHits++
}

if d.dbStatsConfig.TimeStats {
dur = time.Since(now)

if dur > stats.Peak {
stats.Peak = dur
}
stats.Time += dur

if dur > stats.ExistsKeyPatternPeak {
stats.ExistsKeyPatternPeak = dur
}
stats.ExistsKeyPatternTime += dur
}

if d.dbStatsConfig.TableStats {
d.stats.Tables[ts.Name] = stats
} else {
d.stats.AllTables = stats
}

return exists, err
}

////////////////////////////////////////////////////////////////////////////////
// Internal Functions //
////////////////////////////////////////////////////////////////////////////////

var luaScriptExistsKeysPatterns *redis.Script

func init() {
// Register the Lua Script
luaScriptExistsKeysPatterns = redis.NewScript(`
for i,k in pairs(redis.call('KEYS', KEYS[1])) do
return 'true'
end
return 'false'
`)

// Alternate Lua Script
// luaScriptExistsKeysPatterns = redis.NewScript(`
// if #redis.call('KEYS', KEYS[1]) > 0 then
// return 'true'
// else
// return 'false'
// end
// `)

}
128 changes: 128 additions & 0 deletions translib/db/db_keys_pattern_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
////////////////////////////////////////////////////////////////////////////////
// //
// Copyright 2022 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
// its subsidiaries. //
// //
// Licensed under the Apache License, Version 2.0 (the "License"); //
// you may not use this file except in compliance with the License. //
// You may obtain a copy of the License at //
// //
// http://www.apache.org/licenses/LICENSE-2.0 //
// //
// Unless required by applicable law or agreed to in writing, software //
// distributed under the License is distributed on an "AS IS" BASIS, //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and //
// limitations under the License. //
// //
////////////////////////////////////////////////////////////////////////////////

package db

import (
"testing"
)

func BenchmarkExistKeysPattern(b *testing.B) {
for i := 0; i < b.N; i++ {
if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || !exists {
b.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}
}

func BenchmarkGetKeysPattern(b *testing.B) {
for i := 0; i < b.N; i++ {
if _, e := db.GetKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil {
b.Errorf("GetKeysPattern() returns err: %v", e)
}
}
}

func TestGetKeysPattern(t *testing.T) {
if keys, e := db.GetKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || len(keys) == 0 {
t.Errorf("GetKeysPattern() returns len(keys) == 0 || err: %v", e)
}
}

func TestExistKeysPattern(t *testing.T) {
if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || !exists {
t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}

func TestExistKeysPatternSinglular(t *testing.T) {
if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"KEY1"}}); e != nil || !exists {
t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}

func TestExistKeysPatternGeneric(t *testing.T) {
if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"KEY*"}}); e != nil || !exists {
t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}

func TestExistKeysPatternEmpty(t *testing.T) {
if exists, e := db.ExistKeysPattern(&TableSpec{Name: "UNLIKELY_23"},
Key{Comp: []string{"*"}}); e != nil || exists {
t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
}

if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"UNKNOWN"}}); e != nil || exists {
t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
}
}

func TestExistKeysPatternRW(t *testing.T) {
dbRW, err := newDB(ConfigDB)
if err != nil {
t.Fatalf("TestExistKeysPatternRW: newDB() for RW fails err = %v\n", err)
}
t.Cleanup(func() { dbRW.DeleteDB() })

if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || !exists {
t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}

func TestExistKeysPatternSinglularRW(t *testing.T) {
dbRW, err := newDB(ConfigDB)
if err != nil {
t.Fatalf("TestExistKeysPatternSinglularRW: newDB() for RW fails err = %v\n", err)
}
t.Cleanup(func() { dbRW.DeleteDB() })

if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"KEY1"}}); e != nil || !exists {
t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}

func TestExistKeysPatternGenericRW(t *testing.T) {
dbRW, err := newDB(ConfigDB)
if err != nil {
t.Fatalf("TestExistKeysPatternGenericRW: newDB() for RW fails err = %v\n", err)
}
t.Cleanup(func() { dbRW.DeleteDB() })

if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"KEY*"}}); e != nil || !exists {
t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
}
}

func TestExistKeysPatternEmptyRW(t *testing.T) {
dbRW, err := newDB(ConfigDB)
if err != nil {
t.Fatalf("TestExistKeysPatternEmptyRW: newDB() for RW fails err = %v\n", err)
}
t.Cleanup(func() { dbRW.DeleteDB() })

if exists, e := dbRW.ExistKeysPattern(&TableSpec{Name: "UNLIKELY_23"},
Key{Comp: []string{"*"}}); e != nil || exists {
t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
}

if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"UNKNOWN"}}); e != nil || exists {
t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
}
}
Loading

0 comments on commit 42ca0a6

Please sign in to comment.