diff --git a/go.mod b/go.mod index 001aff7e11..afe18e8e74 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.1 github.com/xeipuuv/gojsonschema v1.2.0 + github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 go.mongodb.org/mongo-driver v1.7.1 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0 go.opentelemetry.io/otel v1.7.0 diff --git a/go.sum b/go.sum index 93ad862ae1..c789598107 100644 --- a/go.sum +++ b/go.sum @@ -1578,6 +1578,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 h1:7v7L5lsfw4w8iqBBXETukHo4IPltmD+mWoLRYUmeGN8= +github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869/go.mod h1:Rfzr+sqaDreiCaoQbFCu3sTXxeFq/9kXRuyOoSlGQHE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/cnab/config-adapter/adapter.go b/pkg/cnab/config-adapter/adapter.go index 78ee89638f..b317bc7e04 100644 --- a/pkg/cnab/config-adapter/adapter.go +++ b/pkg/cnab/config-adapter/adapter.go @@ -2,12 +2,15 @@ package configadapter import ( "context" + "encoding/json" "fmt" "path" + "regexp" "strings" "get.porter.sh/porter/pkg/cnab" depsv1 "get.porter.sh/porter/pkg/cnab/dependencies/v1" + depsv2 "get.porter.sh/porter/pkg/cnab/dependencies/v2" "get.porter.sh/porter/pkg/config" "get.porter.sh/porter/pkg/experimental" "get.porter.sh/porter/pkg/manifest" @@ -16,6 +19,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/cnabio/cnab-go/bundle" "github.com/cnabio/cnab-go/bundle/definition" + "github.com/pkg/errors" ) // ManifestConverter converts from a porter manifest to a CNAB bundle definition. @@ -408,33 +412,33 @@ func (c *ManifestConverter) generateBundleImages() map[string]bundle.Image { } func (c *ManifestConverter) generateDependencies() (interface{}, string, error) { - if len(c.Manifest.Dependencies.RequiredDependencies) == 0 { + if len(c.Manifest.Dependencies.Requires) == 0 { return nil, "", nil } // Check if they are using v1 of the dependencies spec or v2 if c.config.IsFeatureEnabled(experimental.FlagDependenciesV2) { - panic("the dependencies-v2 experimental flag was specified but is not yet implemented") + // Ok we are using v2! + deps, err := c.generateDependenciesV2() + return deps, cnab.DependenciesV2ExtensionKey, err } - deps, err := c.generateDependenciesV1() - if err != nil { - return nil, "", err - } + // Default to using v1 of deps + deps := c.generateDependenciesV1() return deps, cnab.DependenciesV1ExtensionKey, nil } -func (c *ManifestConverter) generateDependenciesV1() (*depsv1.Dependencies, error) { - if len(c.Manifest.Dependencies.RequiredDependencies) == 0 { - return nil, nil +func (c *ManifestConverter) generateDependenciesV1() *depsv1.Dependencies { + if len(c.Manifest.Dependencies.Requires) == 0 { + return nil } deps := &depsv1.Dependencies{ - Sequence: make([]string, 0, len(c.Manifest.Dependencies.RequiredDependencies)), - Requires: make(map[string]depsv1.Dependency, len(c.Manifest.Dependencies.RequiredDependencies)), + Sequence: make([]string, 0, len(c.Manifest.Dependencies.Requires)), + Requires: make(map[string]depsv1.Dependency, len(c.Manifest.Dependencies.Requires)), } - for _, dep := range c.Manifest.Dependencies.RequiredDependencies { + for _, dep := range c.Manifest.Dependencies.Requires { dependencyRef := depsv1.Dependency{ Name: dep.Name, Bundle: dep.Bundle.Reference, @@ -454,9 +458,97 @@ func (c *ManifestConverter) generateDependenciesV1() (*depsv1.Dependencies, erro deps.Requires[dep.Name] = dependencyRef } + return deps +} + +func (c *ManifestConverter) generateDependenciesV2() (*depsv2.Dependencies, error) { + deps := &depsv2.Dependencies{ + Requires: make(map[string]depsv2.Dependency, len(c.Manifest.Dependencies.Requires)), + } + + for _, dep := range c.Manifest.Dependencies.Requires { + dependencyRef := depsv2.Dependency{ + Name: dep.Name, + Bundle: dep.Bundle.Reference, + Version: dep.Bundle.Version, + } + + if dep.Bundle.Interface != nil { + if dep.Bundle.Interface.Reference != "" { + dependencyRef.Interface.Reference = dep.Bundle.Interface.Reference + } + if dep.Bundle.Interface.Document != nil { + bundleData, err := json.Marshal(dep.Bundle.Interface.Document) + if err != nil { + return nil, errors.Wrapf(err, "invalid bundle interface document for dependency %s", dep.Name) + } + rawMessage := &json.RawMessage{} + err = rawMessage.UnmarshalJSON(bundleData) + if err != nil { + return nil, errors.Wrapf(err, "could not convert bundle interface document to a raw json message for dependency %s", dep.Name) + } + dependencyRef.Interface.Document = rawMessage + } + } + + if dep.Installation != nil { + dependencyRef.Installation = &depsv2.DependencyInstallation{ + Labels: dep.Installation.Labels, + } + if dep.Installation.Criteria != nil { + dependencyRef.Installation.Criteria = &depsv2.InstallationCriteria{ + MatchInterface: dep.Installation.Criteria.MatchInterface, + MatchNamespace: dep.Installation.Criteria.MatchNamespace, + IgnoreLabels: dep.Installation.Criteria.IgnoreLabels, + } + } + } + + if len(dep.Parameters) > 0 { + dependencyRef.Parameters = make(map[string]depsv2.DependencySource, len(dep.Parameters)) + for param, source := range dep.Parameters { + dependencyRef.Parameters[param] = parseDependencySource(source) + } + } + + if len(dep.Credentials) > 0 { + dependencyRef.Credentials = make(map[string]depsv2.DependencySource, len(dep.Credentials)) + for cred, source := range dep.Credentials { + dependencyRef.Credentials[cred] = parseDependencySource(source) + } + } + + deps.Requires[dep.Name] = dependencyRef + } + return deps, nil } +// TODO: is there a way to feature flag this stuff so that if it's flakey or implemented +// incrementally we can just keep it off? +func parseDependencySource(value string) depsv2.DependencySource { + regex := regexp.MustCompile(`bundle(\.dependencies)?\.([^.]+)\.([^.]+)\.(.+)`) + matches := regex.FindStringSubmatch(value) + if matches == nil || len(matches) < 5 { + return depsv2.DependencySource{Value: value} + } + + dependencyName := matches[2] // bundle.dependencies.DEPENDENCY_NAME + itemType := matches[3] // bundle.dependencies.dependency_name.PARAMETERS.name or bundle.OUTPUTS.name + itemName := matches[4] // bundle.dependencies.dependency_name.parameters.NAME or bundle.outputs.NAME + + result := depsv2.DependencySource{Dependency: dependencyName} + switch itemType { + case "parameters": + result.Parameter = itemName + case "credentials": + result.Credential = itemName + case "outputs": + result.Output = itemName + } + return result +} + func (c *ManifestConverter) generateParameterSources(b *cnab.ExtendedBundle) cnab.ParameterSources { ps := cnab.ParameterSources{} @@ -643,6 +735,8 @@ func (c *ManifestConverter) generateRequiredExtensions(b cnab.ExtendedBundle) [] // Add the appropriate dependencies key if applicable if b.HasDependenciesV1() { requiredExtensions = append(requiredExtensions, cnab.DependenciesV1ExtensionKey) + } else if b.HasDependenciesV2() { + requiredExtensions = append(requiredExtensions, cnab.DependenciesV2ExtensionKey) } // Add the appropriate parameter sources key if applicable diff --git a/pkg/cnab/config-adapter/testdata/porter-depsv2.yaml b/pkg/cnab/config-adapter/testdata/porter-depsv2.yaml new file mode 100644 index 0000000000..f9a5dce638 --- /dev/null +++ b/pkg/cnab/config-adapter/testdata/porter-depsv2.yaml @@ -0,0 +1,45 @@ +schemaVersion: 1.0.0-alpha.1 +name: porter-hello +description: "An example Porter configuration" +version: 0.1.0 +registry: "localhost:5000" + +credentials: + - name: username + description: Name of the database user + required: false + env: ROOT_USERNAME + - name: password + path: /tmp/password + applyTo: + - uninstall + +dependencies: + requires: + - name: mysql + bundle: + reference: "getporter/azure-mysql:5.7" + +mixins: +- exec + +install: +- exec: + description: "Say Hello" + command: bash + flags: + c: echo Hello World + +status: +- exec: + description: "Get World Status" + command: bash + flags: + c: echo The world is on fire + +uninstall: +- exec: + description: "Say Goodbye" + command: bash + flags: + c: echo Goodbye World diff --git a/pkg/cnab/dependencies/v2/types.go b/pkg/cnab/dependencies/v2/types.go new file mode 100644 index 0000000000..764779643f --- /dev/null +++ b/pkg/cnab/dependencies/v2/types.go @@ -0,0 +1,78 @@ +package v2 + +import ( + "encoding/json" +) + +// Dependencies describes the set of custom extension metadata associated with the dependencies spec +// https://github.com/cnabio/cnab-spec/blob/master/500-CNAB-dependencies.md +type Dependencies struct { + // Requires is a list of bundles required by this bundle + Requires map[string]Dependency `json:"requires,omitempty" mapstructure:"requires"` +} + +/* +dependencies: + requires: # dependencies are always created in the current namespace, never global though they can match globally? + mysql: + bundle: + reference: getporter/mysql:v1.0.2 + version: 1.x + interface: # Porter defaults the interface based on usage + reference: getporter/generic-mysql-interface:v1.0.0 # point to an interface bundle to be more specific + bundle: # add extra interface requirements + outputs: + - $id: "mysql-5.7-connection-string" # match on something other than name, so that outputs with different names can be reused + installation: + labels: # labels applied to the installation if created + app: myapp + installation: {{ installation.name }} # exclusive resource + criteria: # criteria for reusing an existing installation, by default must be the same bundle, labels and allows global + matchInterface: true # only match the interface, not the bundle too + matchNamespace: true # must be in the same namespace, disallow global + ignoreLabels: true # allow different labels +*/ + +// Dependency describes a dependency on another bundle +type Dependency struct { + // Name of the dependency + Name string + + // Bundle is the location of the bundle in a registry, for example REGISTRY/NAME:TAG + Bundle string `json:"bundle" mapstructure:"bundle"` + + // Version is a set of allowed versions + Version string `json:"version,omitempty" mapstructure:"version"` + + Interface *DependencyInterface `json:"interface,omitempty" mapstructure:"interface,omitempty"` + + Installation *DependencyInstallation `json:"installation,omitempty" mapstructure:"installation,omitempty"` + + Parameters map[string]DependencySource `json:"parameters,omitempty" mapstructure:"parameters,omitempty"` + Credentials map[string]DependencySource `json:"credentials,omitempty" mapstructure:"credentials,omitempty"` +} + +type DependencySource struct { + Value string `json:"value,omitempty" mapstructure:"value,omitempty"` + Dependency string `json:"dependency,omitempty" mapstructure:"dependency,omitempty"` + Credential string `json:"credential,omitempty" mapstructure:"credential,omitempty"` + Parameter string `json:"parameter,omitempty" mapstructure:"parameter,omitempty"` + Output string `json:"output,omitempty" mapstructure:"output,omitempty"` +} + +type DependencyInstallation struct { + Labels map[string]string `json:"labels,omitempty" mapstructure:"labels,omitempty"` + Criteria *InstallationCriteria `json:"criteria,omitempty" mapstructure:"criteria,omitempty"` +} + +type InstallationCriteria struct { + // MatchInterface specifies if the installation should use the same bundle or just needs to match the interface + MatchInterface bool `json:"matchInterface,omitempty" mapstructure:"matchInterface,omitEmpty"` + MatchNamespace bool `json:"matchNamespace,omitempty" mapstructure:"matchNamespace,omitEmpty"` + IgnoreLabels bool `json:"ignoreLabels,omitempty" mapstructure:"ignoreLabels,omitempty"` +} + +type DependencyInterface struct { + Reference string `json:"reference,omitempty" mapstructure:"reference,omitempty"` + Document *json.RawMessage `json:"document,omitempty" mapstructure:"document,omitempty"` +} diff --git a/pkg/cnab/dependencies_v2.go b/pkg/cnab/dependencies_v2.go new file mode 100644 index 0000000000..11fc7127e1 --- /dev/null +++ b/pkg/cnab/dependencies_v2.go @@ -0,0 +1,81 @@ +package cnab + +import ( + "encoding/json" + + v2 "get.porter.sh/porter/pkg/cnab/dependencies/v2" + "github.com/pkg/errors" +) + +const ( + // DependenciesV2ExtensionShortHand is the short suffix of the DependenciesV2ExtensionKey + DependenciesV2ExtensionShortHand = "dependencies" + + // DependenciesV2ExtensionKey represents the full key for the DependenciesV2Extension. + DependenciesV2ExtensionKey = "sh.porter.dependencies.v2" + + // DependenciesV2Schema represents the schema for the DependenciesV2 Extension + DependenciesV2Schema = "https://porter.sh/extensions/dependencies/v2/schema.json" +) + +// DependenciesV2Extension represents the required extension to enable dependencies +var DependenciesV2Extension = RequiredExtension{ + Shorthand: DependenciesV2ExtensionShortHand, + Key: DependenciesV2ExtensionKey, + Schema: DependenciesV2Schema, + Reader: func(b ExtendedBundle) (interface{}, error) { + return b.DependencyV2Reader() + }, +} + +// ReadDependenciesV2 is a convenience method for returning a bonafide +// DependenciesV2 reference after reading from the applicable section from +// the provided bundle +func (b ExtendedBundle) ReadDependenciesV2() (v2.Dependencies, error) { + raw, err := b.DependencyV2Reader() + if err != nil { + return v2.Dependencies{}, err + } + + deps, ok := raw.(v2.Dependencies) + if !ok { + return v2.Dependencies{}, errors.New("unable to read dependencies v2 extension data") + } + + // Return the dependencies + return deps, nil +} + +// DependencyV2Reader is a Reader for the DependenciesV2Extension, which reads +// from the applicable section in the provided bundle and returns the raw +// data in the form of an interface +func (b ExtendedBundle) DependencyV2Reader() (interface{}, error) { + data, ok := b.Custom[DependenciesV2ExtensionKey] + if !ok { + return nil, errors.Errorf("attempted to read dependencies from bundle but none are defined") + } + + dataB, err := json.Marshal(data) + if err != nil { + return nil, errors.Wrapf(err, "could not marshal the untyped dependencies extension data %q", string(dataB)) + } + + deps := v2.Dependencies{} + err = json.Unmarshal(dataB, &deps) + if err != nil { + return nil, errors.Wrapf(err, "could not unmarshal the dependencies extension %q", string(dataB)) + } + + return deps, nil +} + +// SupportsDependenciesV2 checks if the bundle supports dependencies +func (b ExtendedBundle) SupportsDependenciesV2() bool { + return b.SupportsExtension(DependenciesV2ExtensionKey) +} + +// HasDependenciesV2 returns whether or not the bundle has parameter sources defined. +func (b ExtendedBundle) HasDependenciesV2() bool { + _, ok := b.Custom[DependenciesV2ExtensionKey] + return ok +} diff --git a/pkg/cnab/dependencies_v2_test.go b/pkg/cnab/dependencies_v2_test.go new file mode 100644 index 0000000000..964da231ea --- /dev/null +++ b/pkg/cnab/dependencies_v2_test.go @@ -0,0 +1,79 @@ +package cnab + +import ( + "io/ioutil" + "testing" + + "github.com/cnabio/cnab-go/bundle" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadDependencyV2Properties(t *testing.T) { + t.Parallel() + + data, err := ioutil.ReadFile("testdata/bundle-depsv2.json") + require.NoError(t, err, "cannot read bundle file") + + b, err := bundle.Unmarshal(data) + require.NoError(t, err, "could not unmarshal the bundle") + + bun := ExtendedBundle{*b} + assert.True(t, bun.HasDependenciesV2()) + + deps, err := bun.ReadDependenciesV2() + require.NoError(t, err) + + assert.NotNil(t, deps, "DependenciesV2 was not populated") + assert.Len(t, deps.Requires, 2, "DependenciesV2.Requires is the wrong length") + + dep := deps.Requires["storage"] + assert.NotNil(t, dep, "expected DependenciesV2.Requires to have an entry for 'storage'") + assert.Equal(t, "somecloud/blob-storage", dep.Bundle, "DependencyV2.Bundle is incorrect") + assert.Empty(t, dep.Version, "DependencyV2.Version should be nil") + + dep = deps.Requires["mysql"] + assert.NotNil(t, dep, "expected DependenciesV2.Requires to have an entry for 'mysql'") + assert.Equal(t, "somecloud/mysql", dep.Bundle, "DependencyV2.Bundle is incorrect") + assert.Equal(t, "5.7.x", dep.Version, "DependencyV2.Bundle.Version is incorrect") + +} + +func TestSupportsDependenciesV2(t *testing.T) { + t.Parallel() + + t.Run("supported", func(t *testing.T) { + b := ExtendedBundle{bundle.Bundle{ + RequiredExtensions: []string{DependenciesV2ExtensionKey}, + }} + + assert.True(t, b.SupportsDependenciesV2()) + }) + t.Run("unsupported", func(t *testing.T) { + b := ExtendedBundle{} + + assert.False(t, b.SupportsDependenciesV2()) + }) +} + +func TestHasDependenciesV2(t *testing.T) { + t.Parallel() + + t.Run("has dependencies", func(t *testing.T) { + b := ExtendedBundle{bundle.Bundle{ + RequiredExtensions: []string{DependenciesV2ExtensionKey}, + Custom: map[string]interface{}{ + DependenciesV2ExtensionKey: struct{}{}, + }, + }} + + assert.True(t, b.HasDependenciesV2()) + }) + t.Run("no dependencies", func(t *testing.T) { + b := ExtendedBundle{bundle.Bundle{ + RequiredExtensions: []string{DependenciesV2ExtensionKey}, + }} + + assert.False(t, b.HasDependenciesV2()) + }) +} diff --git a/pkg/cnab/required.go b/pkg/cnab/required.go index 98888eba6f..0221de610b 100644 --- a/pkg/cnab/required.go +++ b/pkg/cnab/required.go @@ -14,6 +14,7 @@ type RequiredExtension struct { // that Porter supports var SupportedExtensions = []RequiredExtension{ DependenciesV1Extension, + DependenciesV2Extension, DockerExtension, FileParameterExtension, ParameterSourcesExtension, diff --git a/pkg/cnab/testdata/bundle-depsv2.json b/pkg/cnab/testdata/bundle-depsv2.json new file mode 100644 index 0000000000..a60f874539 --- /dev/null +++ b/pkg/cnab/testdata/bundle-depsv2.json @@ -0,0 +1,140 @@ +{ + "name": "foo", + "version": "1.0", + "schemaVersion": "99.99", + "invocationImages": [ + { + "imageType": "docker", + "image": "technosophos/helloworld:0.1.0" + } + ], + "images": { + "image1": { + "description": "image1", + "image": "urn:image1uri", + "refs": [ + { + "path": "image1path", + "field": "image.1.field" + } + ] + }, + "image2": { + "name": "image2", + "uri": "urn:image2uri", + "refs": [ + { + "path": "image2path", + "field": "image.2.field" + } + ] + } + }, + "credentials": { + "foo": { + "path": "pfoo" + }, + "bar": { + "env": "ebar" + }, + "quux": { + "path": "pquux", + "env": "equux" + } + }, + "requiredExtensions": [ + "sh.porter.dependencies.v2", + "io.cnab.parameter-sources", + "sh.porter.file-parameters" + ], + "custom": { + "com.example.duffle-bag": { + "icon": "https://example.com/icon.png", + "iconType": "PNG" + }, + "com.example.backup-preferences": { + "enabled": true, + "frequency": "daily" + }, + "sh.porter.dependencies.v2": { + "requires": { + "storage": { + "bundle": "somecloud/blob-storage" + }, + "mysql": { + "bundle": "somecloud/mysql", + "version": "5.7.x" + } + } + }, + "io.cnab.parameter-sources": { + "tfstate": { + "priority": ["output"], + "sources": { + "output": { + "name": "tfstate" + } + } + }, + "mysql_connstr": { + "priority": ["dependencies.output"], + "sources": { + "dependencies.output": { + "dependency": "mysql", + "name": "connstr" + } + } + } + } + }, + "definitions": { + "complexThing": { + "type": "object", + "properties": { + "host": { + "default": "localhost", + "type": "string", + "minLength": 3, + "maxLength": 10 + }, + "port": { + "type": "integer", + "minimum": 8000 + } + }, + "required": [ + "port" + ] + }, + "mysql_connstr": { + "type": "string" + }, + "tfstate": { + "contentEncoding": "base64", + "type": "string" + } + }, + "parameters": { + "serverConfig": { + "definition": "complexThing", + "destination": { + "path": "/cnab/is/go" + } + }, + "tfstate": { + "applyTo": [ "upgrade", "uninstall" ], + "definition": "tfstate", + "required": true + }, + "mysql_connstr": { + "definition": "mysql_connstr" + } + }, + "outputs": { + "tfstate": { + "applyTo": [ "install", "upgrade", "uninstall" ], + "definition": "tfstate", + "path": "/cnab/app/outputs/tfstate" + } + } +} diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index 7bd1e24e49..f683496bdd 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -149,7 +149,7 @@ func (m *Manifest) Validate(cxt *portercontext.Context, strategy schema.CheckStr } } - for _, dep := range m.Dependencies.RequiredDependencies { + for _, dep := range m.Dependencies.Requires { err = dep.Validate(cxt) if err != nil { result = multierror.Append(result, err) @@ -626,17 +626,20 @@ func (mi *MappedImage) ToOCIReference() (cnab.OCIReference, error) { } type Dependencies struct { - RequiredDependencies []*RequiredDependency `yaml:"requires,omitempty"` + Requires []*Dependency `yaml:"requires,omitempty"` } -type RequiredDependency struct { +type Dependency struct { Name string `yaml:"name"` - Bundle BundleCriteria `yaml:"bundle"` - - Parameters map[string]string `yaml:"parameters,omitempty"` + Bundle BundleCriteria `yaml:"bundle"` + Installation *DependencyInstallationConfig `yaml:"installation,omitempty"` + Parameters map[string]string `yaml:"parameters,omitempty"` + Credentials map[string]string `yaml:"credentials,omitempty"` } +type DependencySource string + type BundleCriteria struct { // Reference is the full bundle reference for the dependency // in the format REGISTRY/NAME:TAG @@ -647,20 +650,41 @@ type BundleCriteria struct { // This includes considering prereleases to be invalid if the ranges does not include one. // If you want to have it include pre-releases a simple solution is to include -0 in your range." // https://github.com/Masterminds/semver/blob/master/README.md#checking-version-constraints - Version string `yaml:"version,omitempty"` + Version string `yaml:"version,omitempty"` + Interface *BundleInterface `yaml:"interface,omitempty"` +} + +type BundleInterface struct { + Reference string `yaml:"reference,omitempty"` + Document map[string]interface{} `yaml:"bundle,omitempty"` } -func (d *RequiredDependency) Validate(cxt *portercontext.Context) error { +type DependencyInstallationConfig struct { + Labels map[string]string `yaml:"labels,omitempty"` + Criteria *InstallationCriteria `yaml:"criteria,omitempty"` +} + +type InstallationCriteria struct { + MatchInterface bool `yaml:"matchInterface,omitempty"` + MatchNamespace bool `yaml:"matchNamespace,omitempty"` + IgnoreLabels bool `yaml:"ignoreLabels,omitempty"` +} + +func (d *Dependency) Validate(cxt *portercontext.Context) error { if d.Name == "" { return errors.New("dependency name is required") } if d.Bundle.Reference == "" { - return fmt.Errorf("reference is required for dependency %q", d.Name) - } - - if strings.Contains(d.Bundle.Reference, ":") && len(d.Bundle.Version) > 0 { - return fmt.Errorf("reference for dependency %q can only specify REGISTRY/NAME when version ranges are specified", d.Name) + if d.Bundle.Reference == "" { + return fmt.Errorf("reference is required for dependency %q", d.Name) + } + // TODO: if d.bundle.reference is set, consider the experimental flag is turned on + // perhaps we don't need a flag and can silently enable the functionality based on if it's present instead? + } else { + if strings.Contains(d.Bundle.Reference, ":") && len(d.Bundle.Version) > 0 { + return fmt.Errorf("reference for dependency %q can only specify REGISTRY/NAME when version ranges are specified", d.Name) + } } return nil diff --git a/pkg/manifest/manifest_test.go b/pkg/manifest/manifest_test.go index 489374687c..e82d28af45 100644 --- a/pkg/manifest/manifest_test.go +++ b/pkg/manifest/manifest_test.go @@ -89,28 +89,32 @@ func TestLoadManifestWithDependencies(t *testing.T) { mixin := installStep.GetMixinName() assert.Equal(t, "exec", mixin) - require.Len(t, m.Dependencies.RequiredDependencies, 1, "expected one dependency") - assert.Equal(t, "getporter/azure-mysql:5.7", m.Dependencies.RequiredDependencies[0].Bundle.Reference, "expected a v1 schema for the dependency delcaration") + require.Len(t, m.Dependencies.Requires, 1, "expected one dependency") + assert.Equal(t, "getporter/azure-mysql:5.7", m.Dependencies.Requires[0].Bundle.Reference, "expected a v1 schema for the dependency delcaration") } -func TestLoadManifestWithDependenciesInOrder(t *testing.T) { +func TestLoadManifestWithDependenciesV2(t *testing.T) { c := config.NewTestConfig(t) - c.TestContext.AddTestFile("testdata/porter-with-deps.yaml", config.Name) + c.TestContext.AddTestFile("testdata/porter-depsv2.yaml", config.Name) c.TestContext.AddTestDirectory("testdata/bundles", "bundles") m, err := LoadManifestFrom(context.Background(), c.Config, config.Name) require.NoError(t, err, "could not load manifest") + assert.NotNil(t, m) + assert.Equal(t, []MixinDeclaration{{Name: "exec"}}, m.Mixins) + assert.Len(t, m.Install, 1) + + installStep := m.Install[0] + description, _ := installStep.GetDescription() + require.NotNil(t, description) - nginxDep := m.Dependencies.RequiredDependencies[0] - assert.Equal(t, "nginx", nginxDep.Name) - assert.Equal(t, "localhost:5000/nginx:1.19", nginxDep.Bundle.Reference) + mixin := installStep.GetMixinName() + assert.Equal(t, "exec", mixin) - mysqlDep := m.Dependencies.RequiredDependencies[1] - assert.Equal(t, "mysql", mysqlDep.Name) - assert.Equal(t, "getporter/azure-mysql:5.7", mysqlDep.Bundle.Reference) - assert.Len(t, mysqlDep.Parameters, 1) + require.Len(t, m.Dependencies.Requires, 1, "expected one dependency") + assert.Equal(t, "getporter/azure-mysql:5.7", m.Dependencies.Requires[0].Bundle.Reference, "expected a v2 schema for the dependency delcaration") } diff --git a/pkg/manifest/testdata/porter-depsv2.yaml b/pkg/manifest/testdata/porter-depsv2.yaml new file mode 100644 index 0000000000..3a4cddfbef --- /dev/null +++ b/pkg/manifest/testdata/porter-depsv2.yaml @@ -0,0 +1,37 @@ +schemaVersion: 1.0.0-alpha.1 +name: mybun +version: 0.1.0 +registry: example.com + +mixins: + - exec + +dependencies: + requires: + - name: mysql + bundle: + reference: "getporter/azure-mysql:5.7" + parameters: + database-name: wordpress + +install: + - exec: + command: bash + flags: + c: echo Hello World + +uninstall: + - exec: + description: "Uninstall Hello World" + command: bash + flags: + c: echo Goodbye World + +custom: + foo: bar + +required: + - requiredExtension1 + - requiredExtension2: + config: true + diff --git a/pkg/porter/dependencies.go b/pkg/porter/dependencies.go index 42c3c0869d..c8aac16b43 100644 --- a/pkg/porter/dependencies.go +++ b/pkg/porter/dependencies.go @@ -230,7 +230,7 @@ func (e *dependencyExecutioner) prepareDependency(ctx context.Context, dep *queu } } - for _, manifestDep := range m.Dependencies.RequiredDependencies { + for _, manifestDep := range m.Dependencies.Requires { if manifestDep.Name == dep.Alias { for paramName, value := range manifestDep.Parameters { // Make sure the parameter is defined in the bundle diff --git a/pkg/runtime/runtime_manifest.go b/pkg/runtime/runtime_manifest.go index 9f7dcfeeca..03f50f8e70 100644 --- a/pkg/runtime/runtime_manifest.go +++ b/pkg/runtime/runtime_manifest.go @@ -112,8 +112,8 @@ func (m *RuntimeManifest) GetInstallationName() string { } func (m *RuntimeManifest) loadDependencyDefinitions() error { - m.bundles = make(map[string]cnab.ExtendedBundle, len(m.Dependencies.RequiredDependencies)) - for _, dep := range m.Dependencies.RequiredDependencies { + m.bundles = make(map[string]cnab.ExtendedBundle, len(m.Dependencies.Requires)) + for _, dep := range m.Dependencies.Requires { bunD, err := GetDependencyDefinition(m.config.Context, dep.Name) if err != nil { return err diff --git a/pkg/runtime/runtime_manifest_test.go b/pkg/runtime/runtime_manifest_test.go index 23c5fa829e..991b86faf7 100644 --- a/pkg/runtime/runtime_manifest_test.go +++ b/pkg/runtime/runtime_manifest_test.go @@ -582,28 +582,28 @@ func TestReadManifest_Validate_BundleOutput_Error(t *testing.T) { func TestDependencyV1_Validate(t *testing.T) { testcases := []struct { name string - dep manifest.RequiredDependency + dep manifest.Dependency wantOutput string wantError string }{ { name: "version in reference", - dep: manifest.RequiredDependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: "deislabs/azure-mysql:5.7"}}, + dep: manifest.Dependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: "deislabs/azure-mysql:5.7"}}, wantOutput: "", wantError: "", }, { name: "version ranges", - dep: manifest.RequiredDependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: "deislabs/azure-mysql", Version: "5.7.x-6"}}, + dep: manifest.Dependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: "deislabs/azure-mysql", Version: "5.7.x-6"}}, wantOutput: "", wantError: "", }, { name: "missing reference", - dep: manifest.RequiredDependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: ""}}, + dep: manifest.Dependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: ""}}, wantOutput: "", wantError: `reference is required for dependency "mysql"`, }, { name: "version double specified", - dep: manifest.RequiredDependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: "deislabs/azure-mysql:5.7", Version: "5.7.x-6"}}, + dep: manifest.Dependency{Name: "mysql", Bundle: manifest.BundleCriteria{Reference: "deislabs/azure-mysql:5.7", Version: "5.7.x-6"}}, wantOutput: "", wantError: `reference for dependency "mysql" can only specify REGISTRY/NAME when version ranges are specified`, }, diff --git a/pkg/workflow/bundle_graph.go b/pkg/workflow/bundle_graph.go new file mode 100644 index 0000000000..a13a94ff9d --- /dev/null +++ b/pkg/workflow/bundle_graph.go @@ -0,0 +1,234 @@ +package workflow + +import ( + "context" + + "get.porter.sh/porter/pkg/cnab" + "get.porter.sh/porter/pkg/storage" + "get.porter.sh/porter/pkg/tracing" + "github.com/Masterminds/semver/v3" + "github.com/cnabio/cnab-go/bundle" + "github.com/yourbasic/graph" + "go.opentelemetry.io/otel/attribute" +) + +type BundleGraph struct { + // map[node.key]nodeIndex + nodeKeys map[string]int + nodes []Node + // (DependencyV1 (unresolved), Bundle, Installation) +} + +func NewBundleGraph() *BundleGraph { + return &BundleGraph{ + nodeKeys: make(map[string]int), + } +} + +// RegisterNode adds the specified node to the graph +// returning true if the node is already present. +func (g *BundleGraph) RegisterNode(node Node) bool { + _, exists := g.nodeKeys[node.GetKey()] + if !exists { + nodeIndex := len(g.nodes) + g.nodes = append(g.nodes, node) + g.nodeKeys[node.GetKey()] = nodeIndex + } + return exists +} + +func (g *BundleGraph) Sort() ([]Node, bool) { + dag := graph.New(len(g.nodes)) + for nodeIndex, node := range g.nodes { + for _, depKey := range node.GetRequires() { + depIndex, ok := g.nodeKeys[depKey] + if !ok { + panic("oops") + } + dag.Add(nodeIndex, depIndex) + } + } + + indices, ok := graph.TopSort(dag) + if !ok { + return nil, false + } + + // Reverse the sort so that items with no dependencies are listed first + count := len(indices) + results := make([]Node, count) + for i, nodeIndex := range indices { + results[count-i-1] = g.nodes[nodeIndex] + } + return results, true +} + +func (g *BundleGraph) GetNode(key string) (Node, bool) { + if nodeIndex, ok := g.nodeKeys[key]; ok { + return g.nodes[nodeIndex], true + } + return nil, false +} + +type Node interface { + GetRequires() []string + GetKey() string +} + +var _ Node = BundleNode{} +var _ Node = InstallationNode{} + +type BundleNode struct { + Key string + Reference cnab.BundleReference + Requires []string // TODO: we don't need to know this while resolving, find a less confusing way of storing this so it's clear who should set it +} + +func (d BundleNode) GetKey() string { + return d.Key +} + +func (d BundleNode) GetRequires() []string { + return d.Requires +} + +type InstallationNode struct { + Key string + Namespace string + Name string +} + +func (d InstallationNode) GetKey() string { + return d.Key +} + +func (d InstallationNode) GetRequires() []string { + return nil +} + +type Dependency struct { + Key string + DefaultBundle *BundleReferenceSelector + Interface *BundleInterfaceSelector + InstallationSelector *InstallationSelector + Requires []string +} + +type BundleReferenceSelector struct { + Reference cnab.OCIReference + Version *semver.Constraints +} + +func (s *BundleReferenceSelector) IsMatch(ctx context.Context, inst storage.Installation) bool { + log := tracing.LoggerFromContext(ctx) + log.Debug("Evaluating installation bundle definition") + + if inst.Status.BundleReference == "" { + log.Debug("Installation does not match because it does not have an associated bundle") + return false + } + + ref, err := cnab.ParseOCIReference(inst.Status.BundleReference) + if err != nil { + log.Warn("Could not evaluate installation because the BundleReference is invalid", + attribute.String("reference", inst.Status.BundleReference)) + return false + } + + // If no selector is defined, consider it a match + if s == nil { + return true + } + + // If a version range is specified, ignore the version on the selector and apply the range + // otherwise match the tag or digest + if s.Version != nil { + if inst.Status.BundleVersion == "" { + log.Debug("Installation does not match because it does not have an associated bundle version") + return false + } + + // First check that the repository is the same + gotRepo := ref.Repository() + wantRepo := s.Reference.Repository() + if gotRepo != wantRepo { + log.Warn("Installation does not match because the bundle repository is incorrect", + attribute.String("installation-bundle-repository", gotRepo), + attribute.String("dependency-bundle-repository", wantRepo), + ) + return false + } + + gotVersion, err := semver.NewVersion(inst.Status.BundleVersion) + if err != nil { + log.Warn("Installation does not match because the bundle version is invalid", + attribute.String("installation-bundle-version", inst.Status.BundleVersion), + ) + return false + } + + if s.Version.Check(gotVersion) { + log.Debug("Installation matches because the bundle version is in range", + attribute.String("installation-bundle-version", inst.Status.BundleVersion), + attribute.String("dependency-bundle-version", s.Version.String()), + ) + return true + } else { + log.Debug("Installation does not match because the bundle version is incorrect", + attribute.String("installation-bundle-version", inst.Status.BundleVersion), + attribute.String("dependency-bundle-version", s.Version.String()), + ) + return false + } + } else { + gotRef := ref.String() + wantRef := s.Reference.String() + if gotRef == wantRef { + log.Warn("Installation matches because the bundle reference is correct", + attribute.String("installation-bundle-reference", gotRef), + attribute.String("dependency-bundle-reference", wantRef), + ) + return true + } else { + log.Warn("Installation does not match because the bundle reference is incorrect", + attribute.String("installation-bundle-reference", gotRef), + attribute.String("dependency-bundle-reference", wantRef), + ) + return false + } + } +} + +type InstallationSelector struct { + Bundle *BundleReferenceSelector + Interface *BundleInterfaceSelector + Labels map[string]string + Namespaces []string +} + +func (s InstallationSelector) IsMatch(ctx context.Context, inst storage.Installation) bool { + // Skip checking labels and namespaces, those were used to query the set of + // installations that we are checking + + bundleMatches := s.Bundle.IsMatch(ctx, inst) + if !bundleMatches { + return false + } + + interfaceMatches := s.Interface.IsMatch(ctx, inst) + return interfaceMatches +} + +// BundleInterfaceSelector defines how a bundle is going to be used. +// It is not the same as the bundle definition. +// It works like go interfaces where its defined by its consumer. +type BundleInterfaceSelector struct { + Parameters []bundle.Parameter + Credentials []bundle.Credential + Outputs []bundle.Output +} + +func (s BundleInterfaceSelector) IsMatch(ctx context.Context, inst storage.Installation) bool { + // TODO: implement + return true +} diff --git a/pkg/workflow/bundle_graph_test.go b/pkg/workflow/bundle_graph_test.go new file mode 100644 index 0000000000..a1796d7915 --- /dev/null +++ b/pkg/workflow/bundle_graph_test.go @@ -0,0 +1,70 @@ +package workflow + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" +) + +func TestEngine_DependOnInstallation(t *testing.T) { + /* + A -> B (installation) + A -> C (bundle) + c.parameters.connstr <- B.outputs.connstr + */ + + b := InstallationNode{Key: "b"} + c := BundleNode{ + Key: "c", + Requires: []string{"b"}, + } + a := BundleNode{ + Key: "root", + Requires: []string{"b", "c"}, + } + + g := NewBundleGraph() + g.RegisterNode(a) + g.RegisterNode(b) + g.RegisterNode(c) + sortedNodes, ok := g.Sort() + require.True(t, ok, "graph should not be cyclic") + + gotOrder := make([]string, len(sortedNodes)) + for i, node := range sortedNodes { + gotOrder[i] = node.GetKey() + } + wantOrder := []string{ + "b", + "c", + "root", + } + assert.Equal(t, wantOrder, gotOrder) +} + +/* +✅ need to represent new dependency structure on an extended bundle wrapper +(put in cnab-go later) + +need to read a bundle and make a BundleGraph +? how to handle a param that isn't a pure assignment, e.g. connstr: ${bundle.deps.VM.outputs.ip}:${bundle.deps.SVC.outputs.port} +? when are templates evaluated as the graph is executed (for simplicity, first draft no composition / templating) + +need to resolve dependencies in the graph +* lookup against existing installations +* lookup against semver tags in registry +* lookup against bundle index? when would we look here? (i.e. preferred/registered implementations of interfaces) + +need to turn the sorted nodes into an execution plan +execution plan needs: +* bundle to execute and the installation it will become +* parameters and credentials to pass + * sources: + root parameters/creds + installation outputs + +need to write something that can run an execution plan +* knows how to grab sources and pass them into the bundle +*/ diff --git a/pkg/workflow/default_bundle_resolver.go b/pkg/workflow/default_bundle_resolver.go new file mode 100644 index 0000000000..e6c2a8799c --- /dev/null +++ b/pkg/workflow/default_bundle_resolver.go @@ -0,0 +1,38 @@ +package workflow + +import ( + "context" + + "get.porter.sh/porter/pkg/porter" +) + +var _ DependencyResolver = DefaultBundleResolver{} + +// DefaultBundleResolver resolves the default bundle defined on the dependency. +type DefaultBundleResolver struct { + puller porter.BundleResolver +} + +func (d DefaultBundleResolver) Resolve(ctx context.Context, dep Dependency) (Node, bool, error) { + if dep.DefaultBundle == nil { + return nil, false, nil + } + + pullOpts := porter.BundlePullOptions{ + Reference: dep.DefaultBundle.Reference.String(), + // todo: respect force pull and insecure registry + } + if err := pullOpts.Validate(); err != nil { + return nil, false, err + } + cb, err := d.puller.Resolve(ctx, pullOpts) + if err != nil { + // wrap not found error and indicate that we could resolve anything + return nil, false, err + } + + return BundleNode{ + Key: dep.Key, + Reference: cb.BundleReference, + }, true, nil +} diff --git a/pkg/workflow/engine.go b/pkg/workflow/engine.go new file mode 100644 index 0000000000..7a28075d20 --- /dev/null +++ b/pkg/workflow/engine.go @@ -0,0 +1,169 @@ +package workflow + +import ( + "context" + "fmt" + "github.com/Masterminds/semver/v3" + + "get.porter.sh/porter/pkg/storage" + + "get.porter.sh/porter/pkg/cnab" + depsv2 "get.porter.sh/porter/pkg/cnab/dependencies/v2" + "github.com/pkg/errors" +) + +// Engine handles executing a workflow of bundles to execute. +type Engine struct { + driver WorkflowDriver + resolver DependencyResolver + rootInstallation storage.Installation +} + +// TODO: do we need both a dep graph made up of just bundles (i.e. the unresolved representation) and other with everything resolved (execution plan half filled out)? +func (t Engine) GetDependencyGraph(ctx context.Context, bun cnab.ExtendedBundle) (*BundleGraph, error) { + g := NewBundleGraph() + + // Add the root bundle + root := BundleNode{ + Key: "root", + Reference: cnab.BundleReference{Definition: bun}, + } + + err := t.addBundleToGraph(ctx, g, root) + return g, err +} + +func (t Engine) addBundleToGraph(ctx context.Context, g *BundleGraph, node BundleNode) error { + if exists := g.RegisterNode(node); exists { + // We have already processed this bundle, return to avoid an infinite loop + return nil + } + + bun := node.Reference.Definition + if !bun.HasDependenciesV2() { + return nil + } + + deps, err := bun.ReadDependenciesV2() + if err != nil { + return err + } + + node.Requires = make([]string, 0, len(deps.Requires)) + for depName, dep := range deps.Requires { + depKey := fmt.Sprintf("%s.%s", node.Key, depName) + + resolved, err := t.resolveDependency(ctx, depKey, dep) + if err != nil { + return err + } + + node.Requires = append(node.Requires, depKey) + + depNode, ok := resolved.(BundleNode) + if !ok { + // installations don't have any dependencies so there's nothing left to do + g.RegisterNode(resolved) + continue + } + + requireOutput := func(source depsv2.DependencySource) { + if source.Output == "" { + return + } + + outputRequires := node.Key + if source.Dependency != "" { + outputRequires = node.Key + "." + source.Dependency + } + depNode.Requires = append(depNode.Requires, outputRequires) + } + for _, source := range dep.Parameters { + requireOutput(source) + } + for _, source := range dep.Credentials { + requireOutput(source) + } + t.addBundleToGraph(ctx, g, depNode) + } + + return nil +} + +func (t Engine) resolveDependency(ctx context.Context, name string, dep depsv2.Dependency) (Node, error) { + unresolved := Dependency{Key: name} + if dep.Bundle != "" { + ref, err := cnab.ParseOCIReference(dep.Bundle) + if err != nil { + return nil, errors.Wrapf(err, "invalid bundle for dependency %s", name) + } + unresolved.DefaultBundle = &BundleReferenceSelector{ + Reference: ref, + } + if dep.Version != "" { + unresolved.DefaultBundle.Version, err = semver.NewConstraint(dep.Version) + if err != nil { + return nil, err + } + } + } + + if dep.Interface != nil { + // TODO: convert the interface document into a BundleInterfaceSelector + } + + if dep.Installation != nil { + unresolved.InstallationSelector = &InstallationSelector{} + + matchNamespaces := make([]string, 0, 2) + if !dep.Installation.Criteria.IgnoreLabels { + unresolved.InstallationSelector.Labels = dep.Installation.Labels + } + + matchNamespaces = append(matchNamespaces, t.rootInstallation.Namespace) + if !dep.Installation.Criteria.MatchNamespace && t.rootInstallation.Namespace != "" { + // Include the global namespace + matchNamespaces = append(matchNamespaces, "") + } + unresolved.InstallationSelector.Namespaces = matchNamespaces + + if !dep.Installation.Criteria.MatchInterface { + unresolved.InstallationSelector.Bundle = unresolved.DefaultBundle + } + } + + depNode, resolved, err := t.resolver.Resolve(ctx, unresolved) + if err != nil { + return nil, err + } + + if !resolved { + return nil, errors.Errorf("could not resolve dependency %s", name) + } + + return depNode, nil +} + +func (t Engine) BuildExecutionPlan(ctx context.Context, g *BundleGraph) (ExecutionPlan, error) { + nodes, ok := g.Sort() + if !ok { + return ExecutionPlan{}, fmt.Errorf("could not generate an execution plan, the bundle graph has a cyle") + } + + opts := ExecutionOptions{} + return NewExecutionPlan(nodes, opts), nil +} + +func (t Engine) Execute(ctx context.Context, plan ExecutionPlan) error { + // TODO: for a workflow managed by something external, do we need porter to run the entire time? Can we add a task at the end to update the installation status? + w, err := t.driver.CreateWorkflow(ctx, plan) + if err != nil { + return err + } + + if err = t.driver.StartWorkflow(ctx, w); err != nil { + return err + } + + panic("not implemented") +} diff --git a/pkg/workflow/engine_test.go b/pkg/workflow/engine_test.go new file mode 100644 index 0000000000..3f9d7da824 --- /dev/null +++ b/pkg/workflow/engine_test.go @@ -0,0 +1,68 @@ +package workflow + +import ( + "context" + "get.porter.sh/porter/pkg/experimental" + "testing" + + "get.porter.sh/porter/pkg/config" + + "get.porter.sh/porter/pkg/cnab" + configadapter "get.porter.sh/porter/pkg/cnab/config-adapter" + "get.porter.sh/porter/pkg/manifest" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var _ DependencyResolver = TestResolver{} + +type TestResolver struct { + mocks map[string]Node +} + +func (t TestResolver) Resolve(ctx context.Context, dep Dependency) (Node, bool, error) { + node, ok := t.mocks[dep.Key] + if ok { + return node, true, nil + } + return nil, false, errors.Errorf("no mock exists for %s", dep.Key) +} + +func TestGetDependencyGraphAndSort(t *testing.T) { + c := config.NewTestConfig(t) + c.SetExperimentalFlags(experimental.FlagDependenciesV2) + c.TestContext.UseFilesystem() + ctx := context.Background() + + // load our test porter.yaml into a cnab bundle + m, err := manifest.ReadManifest(c.Context, "testdata/porter.yaml") + require.NoError(t, err) + converter := configadapter.NewManifestConverter(c.Config, m, nil, nil) + bun, err := converter.ToBundle(ctx) + require.NoError(t, err) + + eng := Engine{ + resolver: TestResolver{mocks: map[string]Node{ + "root.load-balancer": InstallationNode{Key: "root.load-balancer"}, + "root.mysql": BundleNode{Key: "root.mysql", Reference: cnab.BundleReference{Definition: cnab.ExtendedBundle{}}}, + }}, + } + g, err := eng.GetDependencyGraph(ctx, bun) + require.NoError(t, err) + + sortedNodes, ok := g.Sort() + require.True(t, ok, "graph should not have a cycle") + + gotOrder := make([]string, len(sortedNodes)) + for i, node := range sortedNodes { + gotOrder[i] = node.GetKey() + } + wantOrder := []string{ + "root.load-balancer", + "root.mysql", + "root", + } + assert.Equal(t, wantOrder, gotOrder) + +} diff --git a/pkg/workflow/execution_plan.go b/pkg/workflow/execution_plan.go new file mode 100644 index 0000000000..04526551cd --- /dev/null +++ b/pkg/workflow/execution_plan.go @@ -0,0 +1,67 @@ +package workflow + +// ExecutionPlan outlines the set of tasks required to execute a bundle +// and indicates when tasks may run in parallel. +type ExecutionPlan struct { + // Ordered list of tasks + Tasks TaskSet + + // debugMode indicates that Porter is going to step through the workflow a task at a time + // This indicates that the workflow driver should generate a workflow definition that supports debugging. + DebugMode bool +} + +type ExecutionOptions struct { + // DebugMode indicates that Porter is going to step through the workflow a task at a time + // This indicates that the workflow driver should generate a workflow definition that supports debugging. + DebugMode bool +} + +func NewExecutionPlan(nodes []Node, opts ExecutionOptions) ExecutionPlan { + return ExecutionPlan{ + Tasks: nil, + DebugMode: opts.DebugMode, + } +} + +// TaskList is an ordered list of tasks. +type TaskList []Task + +// TaskSet contains groups of tasks that can be run in parallel. +type TaskSet []TaskList + +type Task struct { + // Name of the task. Used to refer to a task output + Name string + + // InstallerType defines the type of the installer: docker image, webassembly module, etc. + InstallerType string + + // InstallerReference fully qualified reference to the definition of the installer. + InstallerReference string + + // Inputs given to the task + Inputs []TaskInput + + // Outputs that were generated by the task + Outputs map[string]TaskOutput +} + +type TaskInput struct { + // Env is the name of the environment variable to inject + Env string + + // Path is the full path of the file to inject + Path string + + // Contents of the input value. + Contents string + + // Source where the contents can be resolved. Guaranteed that the source is resolvable when the task is run. + Source string +} + +type TaskOutput struct { + // Path is the full path of the file to collect. + Path string +} diff --git a/pkg/workflow/installation_resolver.go b/pkg/workflow/installation_resolver.go new file mode 100644 index 0000000000..9030067858 --- /dev/null +++ b/pkg/workflow/installation_resolver.go @@ -0,0 +1,124 @@ +package workflow + +import ( + "context" + + "get.porter.sh/porter/pkg/cnab" + "get.porter.sh/porter/pkg/storage" + "go.mongodb.org/mongo-driver/bson" +) + +var _ DependencyResolver = InstallationResolver{} + +// InstallationResolver resolves an existing installation from a dependency +type InstallationResolver struct { + store storage.InstallationStore + + // Namespace of the root installation + namespace string +} + +func (r InstallationResolver) Resolve(ctx context.Context, dep Dependency) (Node, bool, error) { + if dep.InstallationSelector == nil { + return nil, false, nil + } + + // Build a query for matching installations + filter := make(bson.M, 1) + + // Match installations with one of the specified namespaces + namespacesQuery := make([]bson.M, 2) + for _, ns := range dep.InstallationSelector.Namespaces { + namespacesQuery = append(namespacesQuery, bson.M{"namespace": ns}) + } + filter["$or"] = namespacesQuery + + // Match all specified labels + for k, v := range dep.InstallationSelector.Labels { + filter["labels."+k] = v + } + + findOpts := storage.FindOptions{ + Sort: []string{"-namespace", "name"}, + Filter: filter, + } + installations, err := r.store.FindInstallations(ctx, findOpts) + if err != nil { + return nil, false, err + } + + // map[installation index]isMatchBool + matches := make(map[int]bool) + for i, inst := range installations { + if dep.InstallationSelector.IsMatch(ctx, inst) { + matches[i] = true + } + } + + switch len(matches) { + case 0: + return nil, false, nil + case 1: + var instIndex int + for i := range matches { + instIndex = i + } + inst := installations[instIndex] + match := &InstallationNode{ + Key: dep.Key, + Namespace: inst.Namespace, + Name: inst.Name, + } + return match, true, nil + default: + var preferredMatch *storage.Installation + // Prefer an installation that is the same as the default bundle if there are multiple interface matches + if dep.DefaultBundle != nil { + for i, isCandidate := range matches { + if !isCandidate { + continue + } + + inst := installations[i] + bundleRef, err := cnab.ParseOCIReference(inst.Status.BundleReference) + if err != nil { + matches[i] = false + continue + } + + if dep.DefaultBundle.Reference.Repository() == bundleRef.Repository() { + preferredMatch = &inst + break + } + + } + } + + // Prefer an installation in the same namespace if there is both a global and local installation + if preferredMatch != nil && preferredMatch.Namespace == r.namespace { + match := &InstallationNode{ + Key: dep.Key, + Namespace: preferredMatch.Namespace, + Name: preferredMatch.Name, + } + return match, true, nil + } + + // Just pick the first installation sorted by -namespace, name (i.e. global last) + for i, isCandidate := range matches { + if !isCandidate { + continue + } + + inst := installations[i] + match := &InstallationNode{ + Key: dep.Key, + Namespace: inst.Namespace, + Name: inst.Name, + } + return match, true, nil + } + + return nil, false, nil + } +} diff --git a/pkg/workflow/resolver.go b/pkg/workflow/resolver.go new file mode 100644 index 0000000000..529795d38f --- /dev/null +++ b/pkg/workflow/resolver.go @@ -0,0 +1,68 @@ +package workflow + +import ( + "context" + + "get.porter.sh/porter/pkg/storage" + + cnabtooci "get.porter.sh/porter/pkg/cnab/cnab-to-oci" + + "get.porter.sh/porter/pkg/porter" +) + +var _ DependencyResolver = CompositeResolver{} + +type DependencyResolver interface { + Resolve(ctx context.Context, dep Dependency) (Node, bool, error) +} + +// TODO: make a composite resolver that calls all registered child resolvers until a match is found +// installation resolver +// range resolver +// specific bundle resolver +type CompositeResolver struct { + puller porter.BundleResolver + resolvers []DependencyResolver +} + +func NewCompositeResolver(puller porter.BundleResolver, store storage.InstallationStore, registry cnabtooci.RegistryProvider, namespace string) CompositeResolver { + instResolver := InstallationResolver{ + store: store, + namespace: namespace, + } + versionResolver := VersionResolver{ + registry: registry, + } + return CompositeResolver{ + puller: puller, + resolvers: []DependencyResolver{ + instResolver, + versionResolver, + DefaultBundleResolver{}, + }, + } +} + +func (r CompositeResolver) Resolve(ctx context.Context, dep Dependency) (Node, bool, error) { + // pull the default bundle if set, and verify that it meets the interface. It's a problem if it doesn't + // We should stop early if it doesn't work because most likely the interface is defined incorrectly + // We can check at build time that the bundle will work with all the defaults + // don't do this at runtime, assume the bundle has been checked + + // build an interface + // config setting to reuse existing installations + + for _, resolver := range r.resolvers { + depNode, resolved, err := resolver.Resolve(ctx, dep) + if err != nil { + return nil, false, err + } + if resolved { + return depNode, true, nil + } + } + + return nil, false, nil +} + +// TODO: implement the new error source interface and flag it as not found,so we can check for it diff --git a/pkg/workflow/testdata/porter.yaml b/pkg/workflow/testdata/porter.yaml new file mode 100644 index 0000000000..ef65b75e54 --- /dev/null +++ b/pkg/workflow/testdata/porter.yaml @@ -0,0 +1,23 @@ +parameters: + - name: region + type: string + +credentials: + - name: kubeconfig + type: file + +outputs: + - name: connstr + type: string + source: bundle.dependencies.mysql.output.admin-connstr + +dependencies: + requires: + - name: load-balancer + bundle: + reference: example/load-balancer:v1.0.0 + - name: mysql + bundle: + reference: example/mysql:v1.0.0 + parameters: + ip: bundle.dependencies.load-balancer.outputs.ipAddress diff --git a/pkg/workflow/version_resolver.go b/pkg/workflow/version_resolver.go new file mode 100644 index 0000000000..96cb133a2a --- /dev/null +++ b/pkg/workflow/version_resolver.go @@ -0,0 +1,56 @@ +package workflow + +import ( + "context" + "sort" + + "get.porter.sh/porter/pkg/cnab" + cnabtooci "get.porter.sh/porter/pkg/cnab/cnab-to-oci" + "github.com/Masterminds/semver/v3" +) + +var _ DependencyResolver = VersionResolver{} + +// VersionResolver resolves the highest version of the default bundle. +type VersionResolver struct { + registry cnabtooci.RegistryProvider +} + +func (v VersionResolver) Resolve(ctx context.Context, dep Dependency) (Node, bool, error) { + bundle := dep.DefaultBundle + if bundle == nil || bundle.Version == nil { + return nil, false, nil + } + + regOpts := cnabtooci.RegistryOptions{} // TODO: handle passing the registry flags all the way through to here + tags, err := v.registry.ListTags(ctx, bundle.Reference, regOpts) + if err != nil { + return nil, false, err + } + + versions := make(semver.Collection, 0, len(tags)) + for _, tag := range tags { + version, err := semver.NewVersion(tag) + if err == nil { + versions = append(versions, version) + } + } + + if len(versions) == 0 { + return nil, false, nil + } + + sort.Sort(sort.Reverse(versions)) + + // TODO: return the first one that matches the bundle interface + versionRef, err := bundle.Reference.WithTag(versions[0].Original()) + if err != nil { + return nil, false, err + } + + bunRef := cnab.BundleReference{Reference: versionRef} + return BundleNode{ + Key: dep.Key, + Reference: bunRef, + }, true, nil +} diff --git a/pkg/workflow/workflow_driver.go b/pkg/workflow/workflow_driver.go new file mode 100644 index 0000000000..46e53b8b40 --- /dev/null +++ b/pkg/workflow/workflow_driver.go @@ -0,0 +1,24 @@ +package workflow + +import "context" + +// WorkflowDriver is how Porter interacts with workflow drivers, e.g. argo, cadence, etc. +type WorkflowDriver interface { + // CreateWorkflow converts the ExecutionPlan into a definition that the driver understands. + CreateWorkflow(ctx context.Context, plan ExecutionPlan) (WorkflowDefinition, error) + + // StartWorkflow begins the specified workflow. + StartWorkflow(ctx context.Context, workflow WorkflowDefinition) error + + // CancelWorkflow stops the specified workflow. + CancelWorkflow(ctx context.Context, workflow WorkflowDefinition) error + + // RetryWorkflow starts the workflow over at the last failed job(s). + RetryWorkflow(ctx context.Context, workflow WorkflowDefinition) error + + // StepThrough runs only the specified task in the workflow, pausing afterwards so that the workflow can be debugged. + StepThrough(ctx context.Context, workflow WorkflowDefinition, taskName string) error +} + +// WorkflowDefinition is the representation of the ExecutionPlan against a specific workflow driver. +type WorkflowDefinition map[string]interface{}