diff --git a/README.md b/README.md index b04929e..6e2e4f8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # score-go -Reference library for the parsing and loading SCORE files in Go. +Reference library containing common types and functions for building Score implementations in Go. This can be added to your project via: @@ -11,6 +11,13 @@ go get -u github.com/score-spec/score-go@latest **NOTE**: if you project is still using the hand-written types, you will need to stay on `github.com/score-spec/score-go@v0.0.1` and any important fixes to the schema may be back-ported to that branch. +## Packages + +- `github.com/score-spec/score-go/schema` - Go constant with the json schema, and methods for validating a json or yaml structure against the schema. +- `github.com/score-spec/score-go/types` - Go types for Score workloads, services, and resources generated from the json schema. +- `github.com/score-spec/score-go/loader` - Go functions for loading the validated json or yaml structure into a workload struct. +- `github.com/score-spec/score-go/framework` - Common types and functions for Score implementations. + ## Parsing SCORE files This library includes a few utility methods to parse source SCORE files. @@ -19,9 +26,11 @@ This library includes a few utility methods to parse source SCORE files. import ( "os" - "github.com/score-spec/score-go/loader" - "github.com/score-spec/score-go/schema" - score "github.com/score-spec/score-go/types" + "gopkg.in/yaml.v3" + + scoreloader "github.com/score-spec/score-go/loader" + scoreschema "github.com/score-spec/score-go/schema" + scoretypes "github.com/score-spec/score-go/types" ) func main() { @@ -32,20 +41,20 @@ func main() { defer src.Close() var srcMap map[string]interface{} - if err := loader.ParseYAML(&srcMap, src); err != nil { + if err := yaml.NewDecoder(src).Decode(&srcMap); err != nil { panic(err) } - if err := schema.Validate(srcMap); err != nil { + if err := scoreschema.Validate(srcMap); err != nil { panic(err) } - var spec score.Workload - if err := loader.MapSpec(&spec, srcMap); err != nil { + var spec scoretypes.Workload + if err := scoreloader.MapSpec(&spec, srcMap); err != nil { panic(err) } - if err := loader.Normalize(&spec, "."); err != nil { + if err := scoreloader.Normalize(&spec, "."); err != nil { panic(err) } @@ -54,6 +63,10 @@ func main() { } ``` +## Building a Score implementation + +[score-compose](https://github.com/score-spec/score-compose) is the reference Score implementation written in Go and using this library. If you'd like to write a custom Score implementation, use the functions in this library and the `score-compose` implementation as a Guide. + ## Upgrading the schema version When the Score JSON schema is updated in , this repo should be updated to match. diff --git a/framework/override_utils.go b/framework/override_utils.go new file mode 100644 index 0000000..e54908e --- /dev/null +++ b/framework/override_utils.go @@ -0,0 +1,166 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "fmt" + "maps" + "slices" + "strconv" + "strings" +) + +// ParseDotPathParts will parse a common .-separated override path into path elements to traverse. +func ParseDotPathParts(input string) []string { + // support escaping dot's to insert elements with a . in them. + input = strings.ReplaceAll(input, "\\\\", "\x01") + input = strings.ReplaceAll(input, "\\.", "\x00") + parts := strings.Split(input, ".") + for i, part := range parts { + part = strings.ReplaceAll(part, "\x00", ".") + part = strings.ReplaceAll(part, "\x01", "\\") + parts[i] = part + } + return parts +} + +// OverrideMapInMap will take in a decoded json or yaml struct and merge an override map into it. Any maps are merged +// together, other value types are replaced. Nil values will delete overridden keys or otherwise are ignored. This +// returns a shallow copy of the map in a copy-on-write way, so only modified elements are copied. +func OverrideMapInMap(input map[string]interface{}, overrides map[string]interface{}) (map[string]interface{}, error) { + output := maps.Clone(input) + for key, value := range overrides { + if value == nil { + delete(output, key) + continue + } + + existing, hasExisting := output[key] + if !hasExisting { + output[key] = value + continue + } + + eMap, isEMap := existing.(map[string]interface{}) + vMap, isVMap := value.(map[string]interface{}) + if isEMap && isVMap { + output[key], _ = OverrideMapInMap(eMap, vMap) + } else { + output[key] = value + } + } + return output, nil +} + +// OverridePathInMap will take in a decoded json or yaml struct and override a particular path within it with either +// a new value or deletes it. This returns a shallow copy of the map in a copy-on-write way, so only modified elements +// are copied. +func OverridePathInMap(input map[string]interface{}, path []string, isDelete bool, value interface{}) (map[string]interface{}, error) { + return overridePathInMap(input, path, isDelete, value) +} + +func overridePathInMap(input map[string]interface{}, path []string, isDelete bool, value interface{}) (map[string]interface{}, error) { + if len(path) == 0 { + return nil, fmt.Errorf("cannot change root node") + } + + output := maps.Clone(input) + if len(path) == 1 { + if isDelete || value == nil { + delete(output, path[0]) + } else { + output[path[0]] = value + } + return output, nil + } + + if _, ok := output[path[0]]; !ok { + next := make(map[string]interface{}) + subOutput, err := overridePathInMap(next, path[1:], isDelete, value) + if err != nil { + return nil, fmt.Errorf("%s: %w", path[0], err) + } + output[path[0]] = subOutput + return output, nil + } + + switch typed := output[path[0]].(type) { + case map[string]interface{}: + subOutput, err := overridePathInMap(typed, path[1:], isDelete, value) + if err != nil { + return nil, fmt.Errorf("%s: %w", path[0], err) + } + output[path[0]] = subOutput + return output, nil + case []interface{}: + subOutput, err := overridePathInArray(typed, path[1:], isDelete, value) + if err != nil { + return nil, fmt.Errorf("%s: %w", path[0], err) + } + output[path[0]] = subOutput + return output, nil + default: + return nil, fmt.Errorf("%s: cannot set path in non-map/non-array", path[0]) + } +} + +func overridePathInArray(input []interface{}, path []string, isDelete bool, value interface{}) ([]interface{}, error) { + if len(path) == 0 { + return nil, fmt.Errorf("cannot change root node") + } + + pathIndex, err := strconv.Atoi(path[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse '%s' as array index", path[0]) + } + + output := slices.Clone(input) + if len(path) == 1 { + if isDelete || value == nil { + if pathIndex < 0 || pathIndex >= len(input) { + return nil, fmt.Errorf("cannot remove '%d' in array: out of range", pathIndex) + } + return slices.Delete(output, pathIndex, pathIndex+1), nil + } + if pathIndex == -1 { + output = append(output, value) + return output, nil + } + if pathIndex < 0 || pathIndex >= len(input) { + return nil, fmt.Errorf("cannot set '%d' in array: out of range", pathIndex) + } + output[pathIndex] = value + return output, nil + } + + switch typed := output[pathIndex].(type) { + case map[string]interface{}: + subOutput, err := overridePathInMap(typed, path[1:], isDelete, value) + if err != nil { + return nil, fmt.Errorf("%s: %w", path[0], err) + } + output[pathIndex] = subOutput + return output, nil + case []interface{}: + subOutput, err := overridePathInArray(typed, path[1:], isDelete, value) + if err != nil { + return nil, fmt.Errorf("%s: %w", path[0], err) + } + output[pathIndex] = subOutput + return output, nil + default: + return nil, fmt.Errorf("%s: cannot set path in non-map/non-array", path[0]) + } +} diff --git a/framework/override_utils_test.go b/framework/override_utils_test.go new file mode 100644 index 0000000..bf5a6c9 --- /dev/null +++ b/framework/override_utils_test.go @@ -0,0 +1,182 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseDotPathParts(t *testing.T) { + for _, tc := range []struct { + Input string + Expected []string + }{ + {"", []string{""}}, + {"a", []string{"a"}}, + {"a.b", []string{"a", "b"}}, + {"a.-1", []string{"a", "-1"}}, + {"a.b\\.c", []string{"a", "b.c"}}, + {"a.b\\\\.c", []string{"a", "b\\", "c"}}, + } { + t.Run(tc.Input, func(t *testing.T) { + assert.Equal(t, tc.Expected, ParseDotPathParts(tc.Input)) + }) + } +} + +func TestWritePathInStruct(t *testing.T) { + for _, tc := range []struct { + Name string + Spec string + Path []string + Delete bool + Value interface{} + Expected string + ExpectedError error + }{ + { + Name: "simple object set", + Spec: `{"a":{"b":[{}]}}`, + Path: []string{"a", "b", "0", "c"}, + Value: "hello", + Expected: `{"a":{"b":[{"c":"hello"}]}}`, + }, + { + Name: "simple object delete", + Spec: `{"a":{"b":[{"c":"hello"}]}}`, + Path: []string{"a", "b", "0", "c"}, + Delete: true, + Expected: `{"a":{"b":[{}]}}`, + }, + { + Name: "simple array set", + Spec: `{"a":[{}]}`, + Path: []string{"a", "0"}, + Value: "hello", + Expected: `{"a":["hello"]}`, + }, + { + Name: "simple array append", + Spec: `{"a":["hello"]}`, + Path: []string{"a", "-1"}, + Value: "world", + Expected: `{"a":["hello","world"]}`, + }, + { + Name: "simple array delete", + Spec: `{"a":["hello", "world"]}`, + Path: []string{"a", "0"}, + Delete: true, + Expected: `{"a":["world"]}`, + }, + { + Name: "build object via path", + Spec: `{}`, + Path: []string{"a", "b"}, + Value: "hello", + Expected: `{"a":{"b":"hello"}}`, + }, + { + Name: "bad index str", + Spec: `{"a":[]}`, + Path: []string{"a", "b"}, + Value: "hello", + ExpectedError: fmt.Errorf("a: failed to parse 'b' as array index"), + }, + { + Name: "index out of range", + Spec: `{"a": [0]}`, + Path: []string{"a", "2"}, + Value: "hello", + ExpectedError: fmt.Errorf("a: cannot set '2' in array: out of range"), + }, + { + Name: "no append nested arrays", + Spec: `{"a":[[0]]}`, + Path: []string{"a", "0", "-1"}, + Value: "hello", + Expected: `{"a":[[0, "hello"]]}`, + }, + } { + t.Run(tc.Name, func(t *testing.T) { + var inSpec map[string]interface{} + assert.NoError(t, json.Unmarshal([]byte(tc.Spec), &inSpec)) + outSpec, err := OverridePathInMap(inSpec, tc.Path, tc.Delete, tc.Value) + if tc.ExpectedError != nil { + assert.EqualError(t, err, tc.ExpectedError.Error()) + assert.Equal(t, outSpec, map[string]interface{}(nil)) + } else { + if assert.NoError(t, err) { + raw, _ := json.Marshal(outSpec) + assert.JSONEq(t, tc.Expected, string(raw)) + + // verify in spec was not modified + var inSpec2 map[string]interface{} + assert.NoError(t, json.Unmarshal([]byte(tc.Spec), &inSpec2)) + assert.Equal(t, inSpec, inSpec2) + } + } + }) + } +} + +func TestOverrideMapInMap(t *testing.T) { + input := map[string]interface{}{ + "a": "42", + "b": []interface{}{"c", "d"}, + "c": map[string]interface{}{ + "d": "42", + "e": map[string]interface{}{ + "f": "something", + }, + "g": "other", + }, + "h": "thing", + } + stashInput, _ := json.Marshal(input) + output, err := OverrideMapInMap(input, map[string]interface{}{ + "a": "13", + "b": []interface{}{}, + "c": map[string]interface{}{ + "e": map[string]interface{}{ + "z": "thing", + }, + }, + }) + if assert.NoError(t, err) { + assert.Equal(t, map[string]interface{}{ + "a": "13", + "b": []interface{}{}, + "c": map[string]interface{}{ + "d": "42", + "e": map[string]interface{}{ + "f": "something", + "z": "thing", + }, + "g": "other", + }, + "h": "thing", + }, output) + + // verify input was not modified + var inSpec2 map[string]interface{} + assert.NoError(t, json.Unmarshal(stashInput, &inSpec2)) + assert.Equal(t, input, inSpec2) + } +} diff --git a/framework/resource_uid.go b/framework/resource_uid.go new file mode 100644 index 0000000..39f1fdf --- /dev/null +++ b/framework/resource_uid.go @@ -0,0 +1,47 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "fmt" + "strings" +) + +// ResourceUid is a string for a unique resource identifier. This must be constructed through NewResourceUid +type ResourceUid string + +// NewResourceUid constructs a new ResourceUid string. +func NewResourceUid(workloadName string, resName string, resType string, resClass *string, resId *string) ResourceUid { + if resClass == nil { + defaultClass := "default" + resClass = &defaultClass + } + if resId != nil { + return ResourceUid(fmt.Sprintf("%s.%s#%s", resType, *resClass, *resId)) + } + return ResourceUid(fmt.Sprintf("%s.%s#%s.%s", resType, *resClass, workloadName, resName)) +} + +func (r ResourceUid) Type() string { + return string(r)[0:strings.Index(string(r), ".")] +} + +func (r ResourceUid) Class() string { + return string(r)[strings.Index(string(r), ".")+1 : strings.Index(string(r), "#")] +} + +func (r ResourceUid) Id() string { + return string(r)[strings.Index(string(r), "#")+1:] +} diff --git a/framework/resource_uid_test.go b/framework/resource_uid_test.go new file mode 100644 index 0000000..3417509 --- /dev/null +++ b/framework/resource_uid_test.go @@ -0,0 +1,47 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResourceUid_basic(t *testing.T) { + r := NewResourceUid("work", "my-res", "res-type", nil, nil) + assert.Equal(t, "res-type.default#work.my-res", string(r)) + assert.Equal(t, "work.my-res", r.Id()) + assert.Equal(t, "res-type", r.Type()) + assert.Equal(t, "default", r.Class()) +} + +func TestResourceUid_with_class(t *testing.T) { + someClass := "something" + r := NewResourceUid("work", "my-res", "res-type", &someClass, nil) + assert.Equal(t, "res-type.something#work.my-res", string(r)) + assert.Equal(t, "work.my-res", r.Id()) + assert.Equal(t, "res-type", r.Type()) + assert.Equal(t, "something", r.Class()) +} + +func TestResourceUid_with_id(t *testing.T) { + someId := "something" + r := NewResourceUid("work", "my-res", "res-type", nil, &someId) + assert.Equal(t, "res-type.default#something", string(r)) + assert.Equal(t, "something", r.Id()) + assert.Equal(t, "res-type", r.Type()) + assert.Equal(t, "default", r.Class()) +} diff --git a/framework/state.go b/framework/state.go new file mode 100644 index 0000000..9e4a4ce --- /dev/null +++ b/framework/state.go @@ -0,0 +1,291 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "fmt" + "maps" + "reflect" + "slices" + + score "github.com/score-spec/score-go/types" +) + +// State is the mega-structure that contains the state of our workload specifications and resources. +// Score specs are added to this structure and it stores the current resource set. Extra implementation specific fields +// are supported by the generic parameters. +type State[StateExtras any, WorkloadExtras any] struct { + Workloads map[string]ScoreWorkloadState[WorkloadExtras] `yaml:"workloads"` + Resources map[ResourceUid]ScoreResourceState `yaml:"resources"` + SharedState map[string]interface{} `yaml:"shared_state"` + Extras StateExtras `yaml:",inline"` +} + +// NoExtras can be used in place of the state or workload extras if no additional fields are needed. +type NoExtras struct { +} + +// ScoreWorkloadState is the state stored per workload. We store the recorded workload spec, the file it came from if +// necessary to resolve relative references, and any extras for this implementation. +type ScoreWorkloadState[WorkloadExtras any] struct { + // Spec is the final score spec after all overrides and images have been set. This is a validated score file. + Spec score.Workload `yaml:"spec"` + // File is the source score file if known. + File *string `yaml:"file,omitempty"` + // Extras stores any implementation specific extras needed for this workload. + Extras WorkloadExtras `yaml:",inline"` +} + +// ScoreResourceState is the state stored and tracked for each resource. +type ScoreResourceState struct { + // Type is the resource type. + Type string `yaml:"type"` + // Class is the resource class or 'default' if not provided. + Class string `yaml:"class"` + // Id is the generated id for the resource, either . or .. This is tracked so that + // we can deduplicate and work out where a resource came from. + Id string `yaml:"id"` + + Metadata map[string]interface{} `yaml:"metadata"` + Params map[string]interface{} `yaml:"params"` + // SourceWorkload holds the workload name that had the best definition for this resource. "best" is either the + // first one or the one with params defined. + SourceWorkload string `yaml:"source_workload"` + + // ProvisionerUri is the resolved provisioner uri that should be found in the config. This is tracked so that + // we identify which provisioner was used for a particular instance of the resource. + ProvisionerUri string `yaml:"provisioner"` + // State is the internal state local to this resource. It will be persisted to disk when possible. + State map[string]interface{} `yaml:"state"` + + // Outputs is the current set of outputs for the resource. This is the output of calling the provider. It doesn't + // get persisted to disk. + Outputs map[string]interface{} `yaml:"-"` + // OutputLookupFunc is function that allows certain in-process providers to defer any output generation. If this is + // not provided, it will fall back to using what's in the outputs. + OutputLookupFunc OutputLookupFunc `yaml:"-"` +} + +type OutputLookupFunc func(keys ...string) (interface{}, error) + +// WithWorkload returns a new copy of State with the workload added, if the workload already exists with the same name +// then it will be replaced. +// This is not a deep copy, but any writes are executed in a copy-on-write manner to avoid modifying the source. +func (s *State[StateExtras, WorkloadExtras]) WithWorkload(spec *score.Workload, filePath *string, extras WorkloadExtras) (*State[StateExtras, WorkloadExtras], error) { + out := *s + if s.Workloads == nil { + out.Workloads = make(map[string]ScoreWorkloadState[WorkloadExtras]) + } else { + out.Workloads = maps.Clone(s.Workloads) + } + out.Workloads[spec.Metadata["name"].(string)] = ScoreWorkloadState[WorkloadExtras]{ + Spec: *spec, + File: filePath, + Extras: extras, + } + return &out, nil +} + +// WithPrimedResources returns a new copy of State with all workload resources resolved to at least their initial type, +// class and id. New resources will have an empty provider set. Existing resources will not be touched. +// This is not a deep copy, but any writes are executed in a copy-on-write manner to avoid modifying the source. +func (s *State[StateExtras, WorkloadExtras]) WithPrimedResources() (*State[StateExtras, WorkloadExtras], error) { + out := *s + if s.Resources == nil { + out.Resources = make(map[ResourceUid]ScoreResourceState) + } else { + out.Resources = maps.Clone(s.Resources) + } + + primedResourceUids := make(map[ResourceUid]bool) + for workloadName, workload := range s.Workloads { + for resName, res := range workload.Spec.Resources { + resUid := NewResourceUid(workloadName, resName, res.Type, res.Class, res.Id) + if existing, ok := out.Resources[resUid]; !ok { + out.Resources[resUid] = ScoreResourceState{ + Type: resUid.Type(), + Class: resUid.Class(), + Id: resUid.Id(), + Metadata: res.Metadata, + Params: res.Params, + SourceWorkload: workloadName, + State: map[string]interface{}{}, + } + primedResourceUids[resUid] = true + } else if !primedResourceUids[resUid] { + existing.Metadata = res.Metadata + existing.Params = res.Params + existing.SourceWorkload = workloadName + out.Resources[resUid] = existing + primedResourceUids[resUid] = true + } else { + // multiple definitions of the same shared resource, let's check for conflicting params and metadata + if res.Params != nil { + if existing.Params != nil && !reflect.DeepEqual(existing.Params, res.Params) { + return nil, fmt.Errorf("resource '%s': multiple definitions with different params", resUid) + } + existing.Params = res.Params + existing.SourceWorkload = workloadName + } + if res.Metadata != nil { + if existing.Metadata != nil && !reflect.DeepEqual(existing.Metadata, res.Metadata) { + return nil, fmt.Errorf("resource '%s': multiple definitions with different metadata", resUid) + } + existing.Metadata = res.Metadata + } + out.Resources[resUid] = existing + } + } + } + return &out, nil +} + +func (s *State[StateExtras, WorkloadExtras]) getResourceDependencies(workloadName, resName string) (map[ResourceUid]bool, error) { + outMap := make(map[ResourceUid]bool) + res := s.Workloads[workloadName].Spec.Resources[resName] + if res.Params == nil { + return nil, nil + } + _, err := Substitute((map[string]interface{})(res.Params), func(ref string) (string, error) { + parts := SplitRefParts(ref) + if len(parts) > 1 && parts[0] == "resources" { + rr, ok := s.Workloads[workloadName].Spec.Resources[parts[1]] + if ok { + outMap[NewResourceUid(workloadName, parts[1], rr.Type, rr.Class, rr.Id)] = true + } else { + return ref, fmt.Errorf("refers to unknown resource names '%s'", parts[1]) + } + } + return ref, nil + }) + if err != nil { + return nil, fmt.Errorf("workload '%s' resource '%s': %w", workloadName, resName, err) + } + return outMap, nil +} + +// GetSortedResourceUids returns a topological sorting of the resource uids. The output order is deterministic and +// ensures that any resource output placeholder statements are strictly evaluated after their referenced resource. +// If cycles are detected an error will be thrown. +func (s *State[StateExtras, WorkloadExtras]) GetSortedResourceUids() ([]ResourceUid, error) { + + // We're implementing Kahn's algorithm (https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm). + nodesWithNoIncomingEdges := make(map[ResourceUid]bool) + incomingEdges := make(map[ResourceUid]map[ResourceUid]bool, len(s.Resources)) + + // We must first gather all the dependencies of each resource. Many resources won't have dependencies and will go + // straight into the no-incoming-edges set + for workloadName, workload := range s.Workloads { + for resName, res := range workload.Spec.Resources { + deps, err := s.getResourceDependencies(workloadName, resName) + if err != nil { + return nil, err + } + resUid := NewResourceUid(workloadName, resName, res.Type, res.Class, res.Id) + if len(deps) == 0 { + nodesWithNoIncomingEdges[resUid] = true + } else { + incomingEdges[resUid] = deps + } + } + } + + // set up the output list + output := make([]ResourceUid, 0, len(nodesWithNoIncomingEdges)+len(incomingEdges)) + + // now iterate through the nodes with no incoming edges and subtract them from the + for len(nodesWithNoIncomingEdges) > 0 { + + // to get a stable set, we grab whatever is on the set and convert it to a sorted list + subset := make([]ResourceUid, 0, len(nodesWithNoIncomingEdges)) + for uid := range nodesWithNoIncomingEdges { + subset = append(subset, uid) + } + clear(nodesWithNoIncomingEdges) + slices.Sort(subset) + + // we can bulk append the subset to the output + output = append(output, subset...) + + // remove a node from the no-incoming edges set + for _, fromUid := range subset { + // now find any nodes that had an edge going from this node to them + for toUid, m := range incomingEdges { + if m[fromUid] { + // and remove the edge + delete(m, fromUid) + // if there are no incoming edges, then move it to the no-incoming-edges set + if len(m) == 0 { + delete(incomingEdges, toUid) + nodesWithNoIncomingEdges[toUid] = true + } + } + } + } + } + // if we make no progress then there are cycles + if len(incomingEdges) > 0 { + return nil, fmt.Errorf("a cycle exists involving resource param placeholders") + } + return output, nil +} + +// GetResourceOutputForWorkload returns an output function per resource name in the given workload. This is for +// passing into the compose translation context to resolve placeholder references. +// This does not modify the state. +func (s *State[StateExtras, WorkloadExtras]) GetResourceOutputForWorkload(workloadName string) (map[string]OutputLookupFunc, error) { + workload, ok := s.Workloads[workloadName] + if !ok { + return nil, fmt.Errorf("workload '%s': does not exist", workloadName) + } + out := make(map[string]OutputLookupFunc) + + for resName, res := range workload.Spec.Resources { + resUid := NewResourceUid(workloadName, resName, res.Type, res.Class, res.Id) + state, ok := s.Resources[resUid] + if !ok { + return nil, fmt.Errorf("workload '%s': resource '%s' (%s) is not primed", workloadName, resName, resUid) + } + out[resName] = state.OutputLookup + } + return out, nil +} + +// OutputLookup is a function which can traverse an outputs tree to find a resulting key, this defers to the embedded +// output function if it exists. +func (s *ScoreResourceState) OutputLookup(keys ...string) (interface{}, error) { + if s.OutputLookupFunc != nil { + return s.OutputLookupFunc(keys...) + } else if len(keys) == 0 { + return nil, fmt.Errorf("at least one lookup key is required") + } + var resolvedValue interface{} + resolvedValue = s.Outputs + for _, k := range keys { + ok := resolvedValue != nil + if ok { + var mapV map[string]interface{} + mapV, ok = resolvedValue.(map[string]interface{}) + if !ok { + return "", fmt.Errorf("cannot lookup key '%s', context is not a map", k) + } + resolvedValue, ok = mapV[k] + } + if !ok { + return "", fmt.Errorf("key '%s' not found", k) + } + } + return resolvedValue, nil +} diff --git a/framework/state_test.go b/framework/state_test.go new file mode 100644 index 0000000..6ab97cd --- /dev/null +++ b/framework/state_test.go @@ -0,0 +1,402 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + score "github.com/score-spec/score-go/types" +) + +func mustLoadWorkload(t *testing.T, spec string) *score.Workload { + t.Helper() + var raw score.Workload + require.NoError(t, yaml.Unmarshal([]byte(spec), &raw)) + return &raw +} + +func mustAddWorkload(t *testing.T, s *State[NoExtras, NoExtras], spec string) *State[NoExtras, NoExtras] { + t.Helper() + w := mustLoadWorkload(t, spec) + n, err := s.WithWorkload(w, nil, NoExtras{}) + require.NoError(t, err) + return n +} + +func TestWithWorkload(t *testing.T) { + start := new(State[NoExtras, NoExtras]) + + t.Run("one", func(t *testing.T) { + next, err := start.WithWorkload(mustLoadWorkload(t, ` +metadata: + name: example +containers: + hello-world: + image: hi +resources: + foo: + type: thing +`), nil, NoExtras{}) + require.NoError(t, err) + assert.Len(t, start.Workloads, 0) + assert.Len(t, next.Workloads, 1) + assert.Nil(t, next.Workloads["example"].File, nil) + assert.Equal(t, score.Workload{ + Metadata: map[string]interface{}{"name": "example"}, + Containers: map[string]score.Container{"hello-world": {Image: "hi"}}, + Resources: map[string]score.Resource{"foo": {Type: "thing"}}, + }, next.Workloads["example"].Spec) + }) + + t.Run("two", func(t *testing.T) { + next1, err := start.WithWorkload(mustLoadWorkload(t, ` +metadata: + name: example1 +containers: + hello-world: + image: hi +resources: + foo: + type: thing +`), nil, NoExtras{}) + require.NoError(t, err) + next2, err := next1.WithWorkload(mustLoadWorkload(t, ` +metadata: + name: example2 +containers: + hello-world: + image: hi +`), nil, NoExtras{}) + require.NoError(t, err) + + assert.Len(t, start.Workloads, 0) + assert.Len(t, next1.Workloads, 1) + assert.Len(t, next2.Workloads, 2) + }) +} + +func TestWithPrimedResources(t *testing.T) { + start := new(State[NoExtras, NoExtras]) + + t.Run("empty", func(t *testing.T) { + next, err := start.WithPrimedResources() + require.NoError(t, err) + assert.Len(t, next.Resources, 0) + }) + + t.Run("one workload - nominal", func(t *testing.T) { + next := mustAddWorkload(t, start, ` +metadata: {"name": "example"} +resources: + one: + type: thing + two: + type: thing2 + class: banana + three: + type: thing3 + class: apple + id: dog + metadata: + annotations: + foo: bar + params: + color: green + four: + type: thing4 + id: elephant + five: + type: thing4 + id: elephant + metadata: + x: y + params: + color: blue +`) + next, err := next.WithPrimedResources() + require.NoError(t, err) + assert.Len(t, start.Resources, 0) + assert.Equal(t, map[ResourceUid]ScoreResourceState{ + "thing.default#example.one": { + Type: "thing", Class: "default", Id: "example.one", State: map[string]interface{}{}, + SourceWorkload: "example", + }, + "thing2.banana#example.two": { + Type: "thing2", Class: "banana", Id: "example.two", State: map[string]interface{}{}, + SourceWorkload: "example", + }, + "thing3.apple#dog": { + Type: "thing3", Class: "apple", Id: "dog", State: map[string]interface{}{}, + Metadata: map[string]interface{}{"annotations": score.ResourceMetadata{"foo": "bar"}}, + Params: map[string]interface{}{"color": "green"}, + SourceWorkload: "example", + }, + "thing4.default#elephant": { + Type: "thing4", Class: "default", Id: "elephant", State: map[string]interface{}{}, + Metadata: map[string]interface{}{"x": "y"}, + Params: map[string]interface{}{"color": "blue"}, + SourceWorkload: "example", + }, + }, next.Resources) + }) + + t.Run("one workload - diff metadata", func(t *testing.T) { + next := mustAddWorkload(t, start, ` +metadata: {"name": "example"} +resources: + one: + type: thing + id: elephant + metadata: + x: a + two: + type: thing + id: elephant + metadata: + x: y +`) + next, err := next.WithPrimedResources() + require.EqualError(t, err, "resource 'thing.default#elephant': multiple definitions with different metadata") + assert.Len(t, start.Resources, 0) + }) + + t.Run("one workload - diff params", func(t *testing.T) { + next := mustAddWorkload(t, start, ` +metadata: {"name": "example"} +resources: + one: + type: thing + id: elephant + params: + x: a + two: + type: thing + id: elephant + params: + x: y +`) + next, err := next.WithPrimedResources() + require.EqualError(t, err, "resource 'thing.default#elephant': multiple definitions with different params") + assert.Len(t, start.Resources, 0) + }) + + t.Run("two workload - nominal", func(t *testing.T) { + t.Run("one workload - nominal", func(t *testing.T) { + next := mustAddWorkload(t, start, ` +metadata: {"name": "example1"} +resources: + one: + type: thing + two: + type: thing2 + id: dog +`) + next = mustAddWorkload(t, next, ` +metadata: {"name": "example2"} +resources: + one: + type: thing + two: + type: thing2 + id: dog +`) + next, err := next.WithPrimedResources() + require.NoError(t, err) + assert.Len(t, start.Resources, 0) + assert.Len(t, next.Resources, 3) + assert.Equal(t, map[ResourceUid]ScoreResourceState{ + "thing.default#example1.one": { + Type: "thing", Class: "default", Id: "example1.one", State: map[string]interface{}{}, + SourceWorkload: "example1", + }, + "thing.default#example2.one": { + Type: "thing", Class: "default", Id: "example2.one", State: map[string]interface{}{}, + SourceWorkload: "example2", + }, + "thing2.default#dog": { + Type: "thing2", Class: "default", Id: "dog", State: map[string]interface{}{}, + SourceWorkload: "example1", + }, + }, next.Resources) + }) + }) + +} + +func TestGetSortedResourceUids(t *testing.T) { + + t.Run("empty", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + }, nil, NoExtras{}) + assert.NoError(t, err) + ru, err := s.GetSortedResourceUids() + assert.NoError(t, err) + assert.Empty(t, ru) + }) + + t.Run("one", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res": {Type: "thing", Params: map[string]interface{}{}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + ru, err := s.GetSortedResourceUids() + assert.NoError(t, err) + assert.Equal(t, []ResourceUid{"thing.default#eg.res"}, ru) + }) + + t.Run("one cycle", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res": {Type: "thing", Params: map[string]interface{}{"a": "${resources.res.blah}"}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + _, err = s.GetSortedResourceUids() + assert.EqualError(t, err, "a cycle exists involving resource param placeholders") + }) + + t.Run("two unrelated", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res1": {Type: "thing", Params: map[string]interface{}{}}, + "res2": {Type: "thing", Params: map[string]interface{}{}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + ru, err := s.GetSortedResourceUids() + assert.NoError(t, err) + assert.Equal(t, []ResourceUid{"thing.default#eg.res1", "thing.default#eg.res2"}, ru) + }) + + t.Run("two linked", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res1": {Type: "thing", Params: map[string]interface{}{"x": "${resources.res2.blah}"}}, + "res2": {Type: "thing", Params: map[string]interface{}{}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + ru, err := s.GetSortedResourceUids() + assert.NoError(t, err) + assert.Equal(t, []ResourceUid{"thing.default#eg.res2", "thing.default#eg.res1"}, ru) + }) + + t.Run("two cycle", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res1": {Type: "thing", Params: map[string]interface{}{"x": "${resources.res2.blah}"}}, + "res2": {Type: "thing", Params: map[string]interface{}{"y": "${resources.res1.blah}"}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + _, err = s.GetSortedResourceUids() + assert.EqualError(t, err, "a cycle exists involving resource param placeholders") + }) + + t.Run("three linked", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res1": {Type: "thing", Params: map[string]interface{}{"x": "${resources.res2.blah}"}}, + "res2": {Type: "thing", Params: map[string]interface{}{}}, + "res3": {Type: "thing", Params: map[string]interface{}{"x": "${resources.res1.blah}"}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + ru, err := s.GetSortedResourceUids() + assert.NoError(t, err) + assert.Equal(t, []ResourceUid{"thing.default#eg.res2", "thing.default#eg.res1", "thing.default#eg.res3"}, ru) + }) + + t.Run("complex", func(t *testing.T) { + s, err := new(State[NoExtras, NoExtras]).WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Resources: map[string]score.Resource{ + "res1": {Type: "thing", Params: map[string]interface{}{"a": "${resources.res2.blah} ${resources.res3.blah} ${resources.res4.blah} ${resources.res5.blah} ${resources.res6.blah}"}}, + "res2": {Type: "thing", Params: map[string]interface{}{"a": "${resources.res3.blah} ${resources.res4.blah} ${resources.res5.blah} ${resources.res6.blah}"}}, + "res3": {Type: "thing", Params: map[string]interface{}{"a": "${resources.res4.blah} ${resources.res5.blah} ${resources.res6.blah}"}}, + "res4": {Type: "thing", Params: map[string]interface{}{"a": "${resources.res5.blah} ${resources.res6.blah}"}}, + "res5": {Type: "thing", Params: map[string]interface{}{"a": "${resources.res6.blah}"}}, + "res6": {Type: "thing", Params: map[string]interface{}{}}, + }, + }, nil, NoExtras{}) + assert.NoError(t, err) + ru, err := s.GetSortedResourceUids() + assert.NoError(t, err) + assert.Equal(t, []ResourceUid{"thing.default#eg.res6", "thing.default#eg.res5", "thing.default#eg.res4", "thing.default#eg.res3", "thing.default#eg.res2", "thing.default#eg.res1"}, ru) + }) + +} + +type customStateExtras struct { + Fruit string `yaml:"fruit"` +} + +type customWorkloadExtras struct { + Animal string `yaml:"animal"` +} + +func TestCustomExtras(t *testing.T) { + s := new(State[customStateExtras, customWorkloadExtras]) + s.Resources = make(map[ResourceUid]ScoreResourceState) + s.SharedState = make(map[string]interface{}) + s.Extras.Fruit = "apple" + s, _ = s.WithWorkload(&score.Workload{ + Metadata: map[string]interface{}{"name": "eg"}, + Containers: map[string]score.Container{"example": {Image: "foo"}}, + }, nil, customWorkloadExtras{Animal: "bat"}) + + raw, err := yaml.Marshal(s) + assert.NoError(t, err) + var rawOut map[string]interface{} + assert.NoError(t, yaml.Unmarshal(raw, &rawOut)) + assert.Equal(t, map[string]interface{}{ + "workloads": map[string]interface{}{ + "eg": map[string]interface{}{ + "spec": map[string]interface{}{ + "apiVersion": "", + "metadata": map[string]interface{}{"name": "eg"}, + "containers": map[string]interface{}{ + "example": map[string]interface{}{ + "image": "foo", + }, + }, + }, + "animal": "bat", + }, + }, + "resources": map[string]interface{}{}, + "shared_state": map[string]interface{}{}, + "fruit": "apple", + }, rawOut) + + var s2 State[customStateExtras, customWorkloadExtras] + assert.NoError(t, yaml.Unmarshal(raw, &s2)) + assert.Equal(t, "apple", s2.Extras.Fruit) + assert.Equal(t, "bat", s2.Workloads["eg"].Extras.Animal) + assert.Equal(t, &s2, s) +} diff --git a/framework/substitution.go b/framework/substitution.go new file mode 100644 index 0000000..f19c832 --- /dev/null +++ b/framework/substitution.go @@ -0,0 +1,161 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" +) + +var ( + // placeholderRegEx will search for ${...} with any sequence of characters between them. + placeholderRegEx = regexp.MustCompile(`\$(\$|{([^}]*)})`) +) + +func SplitRefParts(ref string) []string { + subRef := strings.Replace(ref, `\.`, "\000", -1) + parts := strings.Split(subRef, ".") + for i, part := range parts { + parts[i] = strings.Replace(part, "\000", ".", -1) + } + return parts +} + +// SubstituteString replaces all matching '${...}' templates in a source string with whatever is returned +// from the inner function. Double $'s are unescaped. +func SubstituteString(src string, inner func(string) (string, error)) (string, error) { + var err error + result := placeholderRegEx.ReplaceAllStringFunc(src, func(str string) string { + // WORKAROUND: ReplaceAllStringFunc(..) does not provide match details + // https://github.com/golang/go/issues/5690 + var matches = placeholderRegEx.FindStringSubmatch(str) + + // SANITY CHECK + if len(matches) != 3 { + err = errors.Join(err, fmt.Errorf("could not find a proper match in previously captured string fragment")) + return src + } + + // support escaped dollars + if matches[1] == "$" { + return matches[1] + } + + result, subErr := inner(matches[2]) + err = errors.Join(err, subErr) + return result + }) + return result, err +} + +// Substitute does the same thing as SubstituteString but recursively through a map. It returns a copy of the original map. +func Substitute(source interface{}, inner func(string) (string, error)) (interface{}, error) { + if source == nil { + return nil, nil + } + switch v := source.(type) { + case string: + return SubstituteString(v, inner) + case map[string]interface{}: + out := make(map[string]interface{}, len(v)) + for k, v := range v { + v2, err := Substitute(v, inner) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + out[k] = v2 + } + return out, nil + case []interface{}: + out := make([]interface{}, len(v)) + for i, i2 := range v { + i3, err := Substitute(i2, inner) + if err != nil { + return nil, fmt.Errorf("%d: %w", i, err) + } + out[i] = i3 + } + return out, nil + default: + return source, nil + } +} + +func mapLookupOutput(ctx map[string]interface{}) func(keys ...string) (interface{}, error) { + return func(keys ...string) (interface{}, error) { + var resolvedValue interface{} + resolvedValue = ctx + for _, k := range keys { + mapV, ok := resolvedValue.(map[string]interface{}) + if !ok { + return "", fmt.Errorf("cannot lookup key '%s', context is not a map", k) + } + resolvedValue, ok = mapV[k] + if !ok { + return "", fmt.Errorf("key '%s' not found", k) + } + } + return resolvedValue, nil + } +} + +func BuildSubstitutionFunction(metadata map[string]interface{}, resources map[string]OutputLookupFunc) func(string) (string, error) { + metadataLookup := mapLookupOutput(metadata) + return func(ref string) (string, error) { + parts := SplitRefParts(ref) + var resolvedValue interface{} + switch parts[0] { + case "metadata": + if len(parts) < 2 { + return "", fmt.Errorf("invalid ref '%s': requires at least a metadata key to lookup", ref) + } + if rv, err := metadataLookup(parts[1:]...); err != nil { + return "", fmt.Errorf("invalid ref '%s': %w", ref, err) + } else { + resolvedValue = rv + } + case "resources": + if len(parts) < 2 { + return "", fmt.Errorf("invalid ref '%s': requires at least a resource name to lookup", ref) + } + rv, ok := resources[parts[1]] + if !ok { + return "", fmt.Errorf("invalid ref '%s': no known resource '%s'", ref, parts[1]) + } else if len(parts) == 2 { + // TODO: deprecate this - this is an annoying and nonsensical legacy thing + return parts[1], nil + } else if rv2, err := rv(parts[2:]...); err != nil { + return "", fmt.Errorf("invalid ref '%s': %w", ref, err) + } else { + resolvedValue = rv2 + } + default: + return "", fmt.Errorf("invalid ref '%s': unknown reference root, use $$ to escape the substitution", ref) + } + + if asString, ok := resolvedValue.(string); ok { + return asString, nil + } + // TODO: work out how we might support other types here in the future + raw, err := json.Marshal(resolvedValue) + if err != nil { + return "", err + } + return string(raw), nil + } +} diff --git a/framework/substitution_test.go b/framework/substitution_test.go new file mode 100644 index 0000000..39569ad --- /dev/null +++ b/framework/substitution_test.go @@ -0,0 +1,154 @@ +// Copyright 2024 Humanitec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + score "github.com/score-spec/score-go/types" +) + +var substitutionFunction func(string) (string, error) + +func init() { + substitutionFunction = BuildSubstitutionFunction(score.WorkloadMetadata{ + "name": "test-name", + "other": map[string]interface{}{"key": "value"}, + "annotations": map[string]interface{}{ + "key.com/foo-bar": "thing", + }, + }, map[string]OutputLookupFunc{ + "env": func(keys ...string) (interface{}, error) { + if len(keys) != 1 { + return nil, fmt.Errorf("fail") + } + return "${" + keys[0] + "}", nil + }, + "db": func(keys ...string) (interface{}, error) { + if len(keys) < 1 { + return nil, fmt.Errorf("fail") + } + return "${DB_" + strings.ToUpper(strings.Join(keys, "_")) + "?required}", nil + }, + "static": mapLookupOutput(map[string]interface{}{"x": "a"}), + }) +} + +func TestSubstitutionFunction(t *testing.T) { + for _, tc := range []struct { + Input string + Expected string + ExpectedError string + }{ + {Input: "missing", ExpectedError: "invalid ref 'missing': unknown reference root, use $$ to escape the substitution"}, + {Input: "metadata.name", Expected: "test-name"}, + {Input: "metadata", ExpectedError: "invalid ref 'metadata': requires at least a metadata key to lookup"}, + {Input: "metadata.other", Expected: "{\"key\":\"value\"}"}, + {Input: "metadata.other.key", Expected: "value"}, + {Input: "metadata.missing", ExpectedError: "invalid ref 'metadata.missing': key 'missing' not found"}, + {Input: "metadata.name.foo", ExpectedError: "invalid ref 'metadata.name.foo': cannot lookup key 'foo', context is not a map"}, + {Input: "metadata.annotations.key\\.com/foo-bar", Expected: "thing"}, + {Input: "resources.env", Expected: "env"}, + {Input: "resources.env.DEBUG", Expected: "${DEBUG}"}, + {Input: "resources.missing", ExpectedError: "invalid ref 'resources.missing': no known resource 'missing'"}, + {Input: "resources.db", Expected: "db"}, + {Input: "resources.db.host", Expected: "${DB_HOST?required}"}, + {Input: "resources.db.port", Expected: "${DB_PORT?required}"}, + {Input: "resources.db.name", Expected: "${DB_NAME?required}"}, + {Input: "resources.db.name.user", Expected: "${DB_NAME_USER?required}"}, + {Input: "resources.static", Expected: "static"}, + {Input: "resources.static.x", Expected: "a"}, + {Input: "resources.static.y", ExpectedError: "invalid ref 'resources.static.y': key 'y' not found"}, + } { + t.Run(tc.Input, func(t *testing.T) { + res, err := substitutionFunction(tc.Input) + if tc.ExpectedError != "" { + assert.EqualError(t, err, tc.ExpectedError) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.Expected, res) + } + }) + } +} + +func TestSubstituteString(t *testing.T) { + for _, tc := range []struct { + Input string + Expected string + ExpectedError string + }{ + {Input: "", Expected: ""}, + {Input: "abc", Expected: "abc"}, + {Input: "$abc", Expected: "$abc"}, + {Input: "abc $$ abc", Expected: "abc $ abc"}, + {Input: "$${abc}", Expected: "${abc}"}, + {Input: "$${abc .4t3298y *(^&(*}", Expected: "${abc .4t3298y *(^&(*}"}, + {Input: "my name is ${metadata.name}", Expected: "my name is test-name"}, + {Input: "my name is ${metadata.thing\\.two}", ExpectedError: "invalid ref 'metadata.thing\\.two': key 'thing.two' not found"}, + {Input: "my name is ${}", ExpectedError: "invalid ref '': unknown reference root, use $$ to escape the substitution"}, + { + Input: "postgresql://${resources.db.user}:${resources.db.password}@${resources.db.host}:${resources.db.port}/${resources.db.name}", + Expected: "postgresql://${DB_USER?required}:${DB_PASSWORD?required}@${DB_HOST?required}:${DB_PORT?required}/${DB_NAME?required}", + }, + } { + t.Run(tc.Input, func(t *testing.T) { + res, err := SubstituteString(tc.Input, substitutionFunction) + if tc.ExpectedError != "" { + if !assert.EqualError(t, err, tc.ExpectedError) { + assert.Equal(t, "", res) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tc.Expected, res) + } + }) + } +} + +func TestSubstituteMap_success(t *testing.T) { + input := map[string]interface{}{ + "a": "1", + "b": "${metadata.name}", + "c": []interface{}{"1", "${metadata.name}", map[string]interface{}{"d": "${metadata.name}"}}, + "d": map[string]interface{}{ + "e": "1", + "f": "${metadata.name}", + }, + } + output, err := Substitute(input, substitutionFunction) + assert.NoError(t, err) + assert.Equal(t, map[string]interface { + }{ + "a": "1", + "b": "test-name", + "c": []interface{}{"1", "test-name", map[string]interface{}{"d": "test-name"}}, + "d": map[string]interface{}{ + "e": "1", + "f": "test-name", + }, + }, output) +} + +func TestSubstituteMap_fail(t *testing.T) { + _, err := Substitute(map[string]interface{}{ + "a": []interface{}{map[string]interface{}{"b": "${metadata.unknown}"}}, + }, substitutionFunction) + assert.EqualError(t, err, "a: 0: b: invalid ref 'metadata.unknown': key 'unknown' not found") +} diff --git a/go.mod b/go.mod index e2cead6..997217d 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,13 @@ module github.com/score-spec/score-go -go 1.21 +go 1.22 + +toolchain go1.22.0 require ( github.com/mitchellh/mapstructure v1.5.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 9cbc3fa..44a92f4 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -7,13 +6,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/loader/loader.go b/loader/loader.go index 0649060..5891480 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -25,6 +25,7 @@ import ( ) // ParseYAML parses YAML into the target mapping structure. +// Deprecated. Please use the yaml/v3 library directly rather than calling this method. func ParseYAML(dest *map[string]interface{}, r io.Reader) error { return yaml.NewDecoder(r).Decode(dest) } @@ -38,6 +39,5 @@ func MapSpec(dest *types.Workload, src map[string]interface{}) error { if err != nil { return fmt.Errorf("initializing decoder: %w", err) } - return mapper.Decode(src) } diff --git a/loader/loader_test.go b/loader/loader_test.go index be2d823..c03d272 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" "github.com/score-spec/score-go/types" ) @@ -236,7 +237,7 @@ resources: var srcMap map[string]interface{} var spec types.Workload - var err = ParseYAML(&srcMap, tt.Source) + var err = yaml.NewDecoder(tt.Source).Decode(&srcMap) if err == nil { err = MapSpec(&spec, srcMap) } diff --git a/schema/validate.go b/schema/validate.go index befa3fd..eef0329 100644 --- a/schema/validate.go +++ b/schema/validate.go @@ -15,6 +15,7 @@ package schema import ( + "bytes" "encoding/json" "fmt" "io" @@ -22,13 +23,14 @@ import ( "github.com/santhosh-tekuri/jsonschema/v5" "gopkg.in/yaml.v3" + + "github.com/score-spec/score-go/types" ) -// Validates source JSON file. -// -// For all vaidation errors returned error would be a *jsonschema.ValidationError. +// ValidateJson validates a json structure read from the given reader source. +// For all validation errors, the returned error would be a *jsonschema.ValidationError. func ValidateJson(r io.Reader) error { - var obj interface{} + var obj map[string]interface{} var dec = json.NewDecoder(r) dec.UseNumber() @@ -39,11 +41,10 @@ func ValidateJson(r io.Reader) error { return Validate(obj) } -// Validates source YAML file. -// -// For all vaidation errors returned error would be a *jsonschema.ValidationError. +// ValidateYaml validates a yaml structure read from the given reader source. +// For all validation errors returned error would be a *jsonschema.ValidationError. func ValidateYaml(r io.Reader) error { - var obj interface{} + var obj map[string]interface{} var dec = yaml.NewDecoder(r) if err := dec.Decode(&obj); err != nil { @@ -53,10 +54,18 @@ func ValidateYaml(r io.Reader) error { return Validate(obj) } -// Validates source structure. -// -// For all vaidation errors returned error would be a *jsonschema.ValidationError. -func Validate(src interface{}) error { +// ValidateSpec validates a workload spec structure by serializing it to yaml and calling ValidateYaml. +func ValidateSpec(spec *types.Workload) error { + intermediate, err := yaml.Marshal(spec) + if err != nil { + return fmt.Errorf("failed to marshal to yaml: %w", err) + } + return ValidateYaml(bytes.NewReader(intermediate)) +} + +// Validate validates the source structure which should be a decoded map. +// For all validation errors returned error would be a *jsonschema.ValidationError. +func Validate(src map[string]interface{}) error { schema, err := jsonschema.CompileString("", ScoreSchemaV1b1) if err != nil { return fmt.Errorf("compiling Score schema: %w", err) diff --git a/schema/validate_test.go b/schema/validate_test.go index eee43c7..6b40628 100644 --- a/schema/validate_test.go +++ b/schema/validate_test.go @@ -20,6 +20,8 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" + + "github.com/score-spec/score-go/types" ) func TestValidateYaml(t *testing.T) { @@ -407,6 +409,28 @@ containers: assert.EqualError(t, err, "jsonschema: '/metadata' does not validate with https://score.dev/schemas/score#/properties/metadata/required: missing properties: 'name'") } +func TestValidateWorkload_nominal(t *testing.T) { + assert.NoError(t, ValidateSpec(&types.Workload{ + ApiVersion: "score.dev/v1b1", + Metadata: map[string]interface{}{ + "name": "my-workload", + }, + Containers: map[string]types.Container{ + "example": {Image: "busybox"}, + }, + })) +} + +func TestValidateWorkload_error(t *testing.T) { + assert.EqualError(t, ValidateSpec(&types.Workload{ + ApiVersion: "score.dev/v1b1", + Metadata: map[string]interface{}{}, + Containers: map[string]types.Container{ + "example": {Image: "busybox"}, + }, + }), "jsonschema: '/metadata' does not validate with https://score.dev/schemas/score#/properties/metadata/required: missing properties: 'name'") +} + func TestApplyCommonUpgradeTransforms(t *testing.T) { var source = []byte(` ---