diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e173da46f..c00548124 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -134,7 +134,12 @@ jobs: run: make build - name: Run flagd binary in background - run: ./bin/flagd start -f file:${{ github.workspace }}/test-harness/symlink_testing-flags.json & + run: | + ./bin/flagd start \ + -f file:${{ github.workspace }}/test-harness/flags/testing-flags.json \ + -f file:${{ github.workspace }}/test-harness/flags/custom-ops.json \ + -f file:${{ github.workspace }}/test-harness/flags/evaluator-refs.json \ + -f file:${{ github.workspace }}/test-harness/flags/zero-flags.json & - name: Run evaluation test suite - run: go clean -testcache && go test -cover ./flagd/tests/integration -run TestEvaluation + run: go clean -testcache && go test -cover ./flagd/tests/integration diff --git a/.gitmodules b/.gitmodules index 61d2eb45d..643f35ec4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "test-harness"] path = test-harness url = https://github.com/open-feature/test-harness.git +[submodule "spec"] + path = spec + url = https://github.com/open-feature/spec.git diff --git a/Makefile b/Makefile index c1370e638..d640b6082 100644 --- a/Makefile +++ b/Makefile @@ -41,9 +41,8 @@ test: # default to core make test-core test-core: go test -race -covermode=atomic -cover -short ./core/pkg/... -coverprofile=core-coverage.out -flagd-integration-test: # dependent on ./bin/flagd start -f file:test-harness/symlink_testing-flags.json +flagd-integration-test: # dependent on ./bin/flagd start -f file:test-harness/flags/testing-flags.json -f file:test-harness/flags/custom-ops.json -f file:test-harness/flags/evaluator-refs.json -f file:test-harness/flags/zero-flags.json go test -cover ./flagd/tests/integration $(ARGS) - cd test-harness; git restore testing-flags.json # reset testing-flags.json run: # default to flagd make run-flagd run-flagd: diff --git a/flagd/go.mod b/flagd/go.mod index a7ed7695f..5e622cefd 100644 --- a/flagd/go.mod +++ b/flagd/go.mod @@ -7,11 +7,12 @@ require ( github.com/dimiro1/banner v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/open-feature/flagd/core v0.6.5 - github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.13 - github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.3 + github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.16 + github.com/open-feature/go-sdk-contrib/tests/flagd v1.3.0 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.16.0 go.uber.org/zap v1.25.0 + ) require ( @@ -49,7 +50,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-memdb v1.3.4 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -66,7 +67,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/open-feature/go-sdk v1.4.0 // indirect + github.com/open-feature/go-sdk v1.7.0 // indirect github.com/open-feature/open-feature-operator v0.2.36 // indirect github.com/open-feature/schemas v0.2.8 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect @@ -76,7 +77,6 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/robfig/cron v1.2.0 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/cors v1.10.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -103,7 +103,7 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.13.0 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.15.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect diff --git a/flagd/go.sum b/flagd/go.sum index 2cf16b1b8..c5ca9959a 100644 --- a/flagd/go.sum +++ b/flagd/go.sum @@ -614,8 +614,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= +github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= @@ -678,18 +678,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/open-feature/flagd/core v0.6.3 h1:9SNJh5EtO2oWy15cZf3Mm36DazW14TOaDsWyzitU9SU= -github.com/open-feature/flagd/core v0.6.3/go.mod h1:vVk5iF8+4htFC18Bk65acLYqIOaqk/OB4T7Xn87kyps= -github.com/open-feature/flagd/core v0.6.4 h1:MLdo2XaLgB2sexXNeIAELYfFlmEUTNF4C58KvdXshXU= -github.com/open-feature/flagd/core v0.6.4/go.mod h1:RAMKLdX0umV4OMT+rU2DNFO8bUiSiYQmpbCoBHvgImE= github.com/open-feature/flagd/core v0.6.5 h1:fXTLAbguK/XFNYt9G+7FBjqnu8QRXuwsy4+057Y41k0= github.com/open-feature/flagd/core v0.6.5/go.mod h1:h9pnlZnI9pe5Dor2ro9jHGRyRSrZvUve7vOvZtxtW/A= -github.com/open-feature/go-sdk v1.4.0 h1:QvFXywCvm9J6uS8+kESHnzi5sLtRkxwhS/ooS7SaF8M= -github.com/open-feature/go-sdk v1.4.0/go.mod h1:DtxB5FPc2GDjOZRFXm+hEjB+JPC8GTYCKo9JuKNx1F0= -github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.13 h1:vw9JzZInncUmhnozKoQsq/CrskROtU2K8TIgZZ69NLE= -github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.13/go.mod h1:AuVhEbeSlK1SFMFezq5eafTv67r10udiarTgaA3ekow= -github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.3 h1:yHBoQ0TxBosI48aclqJbIt1EigJIjKpVtRlz9uD3RpI= -github.com/open-feature/go-sdk-contrib/tests/flagd v1.2.3/go.mod h1:S27vygP91PapxWCxQhJf29v+w6RCMKp9Q/r4FpJnq88= +github.com/open-feature/go-sdk v1.7.0 h1:g6o/sf5xaED7xawXwZ0LqE7RS9co7HZJMClXCXmIbgw= +github.com/open-feature/go-sdk v1.7.0/go.mod h1:5aV26tEoQuompRa8A6BAGukhi88pRMZsslJJwucGNzs= +github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.16 h1:Itgdo1iSCZkHQ3A6dOzpp8fZU4MRqm9/5HiUO1+bkaQ= +github.com/open-feature/go-sdk-contrib/providers/flagd v0.1.16/go.mod h1:inzptmy90VfLUeKyTfqsb2/19u+Heu/ylKuCPl8mvBs= +github.com/open-feature/go-sdk-contrib/tests/flagd v1.3.0 h1:XstvqjyOAo7s0no2z7R8kN1TrIRWzXtu867h0O+gzkk= +github.com/open-feature/go-sdk-contrib/tests/flagd v1.3.0/go.mod h1:Hb/nrUmwLdo4j+r+WwdBR0jXFaVrKdbVOHaQgq6ra2A= github.com/open-feature/open-feature-operator v0.2.36 h1:dzyZh9JSIRvXkfpM9ynYplNk7vjQFLs9sd5aHhF48z4= github.com/open-feature/open-feature-operator v0.2.36/go.mod h1:nM7T4oGQukeGmcAFkQm0uwt8WFdDb5hYPjXkm7pHhX4= github.com/open-feature/schemas v0.2.8 h1:oA75hJXpOd9SFgmNI2IAxWZkwzQPUDm7Jyyh3q489wM= @@ -718,9 +714,6 @@ github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfm github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= -github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/cors v1.10.0 h1:62NOS1h+r8p1mW6FM0FSB0exioXLhd/sh15KpjWBZ+8= github.com/rs/cors v1.10.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -837,8 +830,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -949,8 +942,6 @@ golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1309,13 +1300,9 @@ google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1356,8 +1343,6 @@ google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= diff --git a/flagd/tests/integration/caching_test.go b/flagd/tests/integration/caching_test.go deleted file mode 100644 index 6daa381b4..000000000 --- a/flagd/tests/integration/caching_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package integration_test - -import ( - "flag" - "testing" - - "github.com/cucumber/godog" - flagd "github.com/open-feature/go-sdk-contrib/providers/flagd/pkg" - "github.com/open-feature/go-sdk-contrib/tests/flagd/pkg/integration" -) - -const ( - flagConfigurationPath = "../../../test-harness/symlink_testing-flags.json" - flagConfigurationTargetFilePath = "../../../test-harness/testing-flags.json" - flagConfigurationMutatedPath = "../../../test-harness/mutated-testing-flags.json" -) - -func TestCaching(t *testing.T) { - if testing.Short() { - t.Skip() - } - - flag.Parse() - - var providerOptions []flagd.ProviderOption - name := "caching.feature" - - if tls == "true" { - name = "caching_tls.feature" - providerOptions = []flagd.ProviderOption{flagd.WithTLS(certPath)} - } - - initializeCachingScenario, err := integration.InitializeCachingScenario( - flagConfigurationTargetFilePath, flagConfigurationPath, flagConfigurationMutatedPath, providerOptions...) - if err != nil { - t.Fatal(err) - } - - testSuite := godog.TestSuite{ - Name: name, - ScenarioInitializer: initializeCachingScenario, - Options: &godog.Options{ - Format: "pretty", - Paths: []string{"../../../test-harness/features/caching.feature"}, - TestingT: t, // Testing instance that will run subtests. - }, - } - - if testSuite.Run() != 0 { - t.Fatal("non-zero status returned, failed to run caching tests") - } -} diff --git a/flagd/tests/integration/evaluation_test.go b/flagd/tests/integration/evaluation_test.go index fe72add16..fec27b0cb 100644 --- a/flagd/tests/integration/evaluation_test.go +++ b/flagd/tests/integration/evaluation_test.go @@ -29,7 +29,7 @@ func TestEvaluation(t *testing.T) { ScenarioInitializer: integration.InitializeEvaluationScenario(providerOptions...), Options: &godog.Options{ Format: "pretty", - Paths: []string{"../../../test-harness/features/evaluation.feature"}, + Paths: []string{"../../../spec/specification/assets/gherkin/evaluation.feature"}, TestingT: t, // Testing instance that will run subtests. Strict: true, }, diff --git a/flagd/tests/integration/json_evaluator_test.go b/flagd/tests/integration/json_evaluator_test.go new file mode 100644 index 000000000..22b01fa38 --- /dev/null +++ b/flagd/tests/integration/json_evaluator_test.go @@ -0,0 +1,36 @@ +package integration_test + +import ( + "flag" + "testing" + + "github.com/cucumber/godog" + flagd "github.com/open-feature/go-sdk-contrib/providers/flagd/pkg" + "github.com/open-feature/go-sdk-contrib/tests/flagd/pkg/integration" +) + +func TestJsonEvaluator(t *testing.T) { + if testing.Short() { + t.Skip() + } + + flag.Parse() + + var providerOptions []flagd.ProviderOption + name := "flagd-json-evaluator.feature" + + testSuite := godog.TestSuite{ + Name: name, + ScenarioInitializer: integration.InitializeFlagdJsonScenario(providerOptions...), + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"../../../test-harness/gherkin/flagd-json-evaluator.feature"}, + TestingT: t, // Testing instance that will run subtests. + Strict: true, + }, + } + + if testSuite.Run() != 0 { + t.Fatal("non-zero status returned, failed to run evaluation tests") + } +} diff --git a/spec b/spec new file mode 160000 index 000000000..a0b377790 --- /dev/null +++ b/spec @@ -0,0 +1 @@ +Subproject commit a0b37779091bd2bcce5ba3505ca52c5d26075e67 diff --git a/test-harness b/test-harness index 48c56d131..6493d000b 160000 --- a/test-harness +++ b/test-harness @@ -1 +1 @@ -Subproject commit 48c56d1314ba89f6d1fdc2c17a8e4ac42e6b1981 +Subproject commit 6493d000b6d5b35c2636a088ad21cff03c2d6281