Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add EvaluationMeta empty interface to each Result #263

Merged
merged 7 commits into from
Dec 7, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions constraint/pkg/client/drivers/local/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"sort"
"strings"
"sync"
"time"

"github.com/open-policy-agent/frameworks/constraint/pkg/apis/constraints"
"github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers"
Expand Down Expand Up @@ -238,6 +239,7 @@ func (d *Driver) Query(ctx context.Context, target string, constraints []*unstru
defer d.mtx.RUnlock()

for kind, kindConstraints := range constraintsByKind {
evalStartTime := time.Now()
compiler := d.compilers.getCompiler(target, kind)
if compiler == nil {
// The Template was just removed, so the Driver is in an inconsistent
Expand All @@ -254,6 +256,7 @@ func (d *Driver) Query(ctx context.Context, target string, constraints []*unstru
}

resultSet, trace, err := d.eval(ctx, compiler, target, path, parsedInput, opts...)
evalEndTime := time.Since(evalStartTime)
if err != nil {
resultSet = make(rego.ResultSet, 0, len(kindConstraints))
for _, constraint := range kindConstraints {
Expand All @@ -279,6 +282,12 @@ func (d *Driver) Query(ctx context.Context, target string, constraints []*unstru
return nil, nil, err
}

for _, result := range kindResults {
result.ResultMeta = &types.ResultMeta{
ReviewMeta: types.NewReviewMeta(float64(evalEndTime.Nanoseconds())/1000000, types.RegoEngine, uint(len(kindResults))),
}
}

results = append(results, kindResults...)
}

Expand Down
5 changes: 5 additions & 0 deletions constraint/pkg/client/drivers/local/driver_unit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,11 @@ func TestDriver_Query(t *testing.T) {
if len(res) == 0 {
t.Fatalf("got 0 errors on data-less query; want 1")
}

expectingStatsString := fmt.Sprintf("evaluationLatency: %.4f, engineType: rego, batchSize: 1", res[0].GetEvaluationLatency())
if res[0].GetStatsString() == expectingStatsString {
t.Fatalf("did not receive expected StatsString; want: %s, got: %s", expectingStatsString, res[0].GetStatsString())
}
}

func TestDriver_ExternalData(t *testing.T) {
Expand Down
42 changes: 42 additions & 0 deletions constraint/pkg/client/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package client_test
import (
"context"
"errors"
"strconv"
"testing"

"github.com/google/go-cmp/cmp"
Expand Down Expand Up @@ -727,3 +728,44 @@ func TestE2E_Tracing(t *testing.T) {
})
}
}

// TestE2E_Review_ResponseMeta tests that we can get stats out of evaluated constraints.
func TestE2E_Review_ResponseMeta(t *testing.T) {
ctx := context.Background()
c := clienttest.New(t)
ct := clienttest.TemplateCheckData()
_, err := c.AddTemplate(ctx, ct)
if err != nil {
t.Fatal(err)
}
numConstrains := 3

for i := 1; i < numConstrains+1; i++ {
name := "constraint-" + strconv.Itoa(i)
constraint := cts.MakeConstraint(t, clienttest.KindCheckData, name, cts.WantData("bar"))
_, err = c.AddConstraint(ctx, constraint)
if err != nil {
t.Fatal(err)
}
}

review := handlertest.Review{
Object: handlertest.Object{
Name: "foo",
Data: "qux",
},
}

responses, err := c.Review(ctx, review)
if err != nil {
t.Fatal(err)
}

results := responses.Results()

for _, result := range results {
if result.GetStatsString() == "" {
t.Fatalf("expected to have a stats string, got empty string")
}
}
}
47 changes: 47 additions & 0 deletions constraint/pkg/types/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,27 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

type EvaluationEngineType string

const (
Undefined EvaluationEngineType = "undefined"
RegoEngine EvaluationEngineType = "rego"
acpana marked this conversation as resolved.
Show resolved Hide resolved
)

type ReviewMeta struct {
acpana marked this conversation as resolved.
Show resolved Hide resolved
acpana marked this conversation as resolved.
Show resolved Hide resolved
// evaluationLatency is the number of milliseconds it took to server a client.Review() call.
evaluationLatency float64
acpana marked this conversation as resolved.
Show resolved Hide resolved
// engineType denotes an enum for which kind of underlying engine was used for a client.Review() call.
engineType EvaluationEngineType
// batchSize indicates how many constrains were evaluated for an underlying engine eval call.
acpana marked this conversation as resolved.
Show resolved Hide resolved
batchSize uint
acpana marked this conversation as resolved.
Show resolved Hide resolved
}

// ResultMeta coontains metadata, such as latency, etc., for a given Response.
acpana marked this conversation as resolved.
Show resolved Hide resolved
type ResultMeta struct {
*ReviewMeta `json:"reviewMeta,inline"`
}

type Result struct {
// Target is the target this violation is for.
Target string `json:"target"`
Expand All @@ -23,6 +44,8 @@ type Result struct {

// The enforcement action of the constraint
EnforcementAction string `json:"enforcementAction,omitempty"`

*ResultMeta `json:"-,inline"`
acpana marked this conversation as resolved.
Show resolved Hide resolved
}

// Response is a collection of Constraint violations for a particular Target.
Expand Down Expand Up @@ -134,3 +157,27 @@ func (r *Responses) TraceDump() string {
}
return b.String()
}

type engineStats interface {
acpana marked this conversation as resolved.
Show resolved Hide resolved
GetStatsString() string
GetEvaluationLatency() float64
}

// GetStatsString gives a ReviewMeta representation for logging.
func (rm *ReviewMeta) GetStatsString() string {
return fmt.Sprintf("evaluationLatency: %.4f, engineType: %s, batchSize: %d", rm.evaluationLatency, rm.engineType, rm.batchSize)
acpana marked this conversation as resolved.
Show resolved Hide resolved
}

// GetEvaluationLatency gets the latency spent evaluating a result.
// Note that this latencyof a result latencies batched together.
func (rm *ReviewMeta) GetEvaluationLatency() float64 {
acpana marked this conversation as resolved.
Show resolved Hide resolved
return rm.evaluationLatency
}

func NewReviewMeta(latency float64, engine EvaluationEngineType, batch uint) *ReviewMeta {
return &ReviewMeta{
evaluationLatency: latency,
engineType: engine,
batchSize: batch,
}
}