Skip to content
This repository has been archived by the owner on Jul 4, 2024. It is now read-only.

Added resource annotations and parameters support #39

Merged
merged 5 commits into from
May 23, 2023
Merged

Added resource annotations and parameters support #39

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
80f216c
feat: added resource annotations support
ghen Apr 18, 2023
0f98c26
feat: added resource paramenters support
ghen Apr 18, 2023
af6aebe
feat: override humanitec resource id with annotations
ghen Apr 19, 2023
a4de2fc
fix: updated warning message for deprecated resource extensions
ghen Apr 21, 2023
31ce21a
fix: added support for fully qualified resource references via annota…
ghen May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ require (
github.com/golang/mock v1.6.0
github.com/imdario/mergo v0.3.13
github.com/mitchellh/mapstructure v1.5.0
github.com/score-spec/score-go v0.0.0-20221019054335-3510902b5f8b
github.com/score-spec/score-go v0.0.0-20230417150859-c1bf3fbe372b
github.com/sendgrid/rest v2.6.9+incompatible
github.com/spf13/cobra v1.6.0
github.com/stretchr/testify v1.8.0
github.com/stretchr/testify v1.8.1
gopkg.in/yaml.v3 v3.0.1
)

Expand Down
8 changes: 5 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/score-spec/score-go v0.0.0-20221019054335-3510902b5f8b h1:Ws6TNwu+OuoR+K7C3fKLCuaeyAiGU0G7Xkaz9UUygWA=
github.com/score-spec/score-go v0.0.0-20221019054335-3510902b5f8b/go.mod h1:eNU0evgibNfV6ESUfRKjWcfGPmd92dI8dsUN/GBouZs=
github.com/score-spec/score-go v0.0.0-20230417150859-c1bf3fbe372b h1:s6DuDF4QC/jjXvBYYJfYKaRgsTGc3dTNwbT43eJCC04=
github.com/score-spec/score-go v0.0.0-20230417150859-c1bf3fbe372b/go.mod h1:eNU0evgibNfV6ESUfRKjWcfGPmd92dI8dsUN/GBouZs=
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=
github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
Expand All @@ -23,9 +23,11 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down
75 changes: 54 additions & 21 deletions internal/humanitec/convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ package humanitec

import (
"fmt"
"log"
"strings"

mergo "github.com/imdario/mergo"
Expand All @@ -17,6 +18,10 @@ import (
humanitec "github.com/score-spec/score-humanitec/internal/humanitec_go/types"
)

const (
AnnotationLabelResourceId = "score.humanitec.io/resId"
)

// getProbeDetails extracts an httpGet probe details from the source spec.
// Returns nil if the source spec is empty.
func getProbeDetails(probe *score.ContainerProbeSpec) map[string]interface{} {
Expand Down Expand Up @@ -157,36 +162,62 @@ func ConvertSpec(name, envID string, spec *score.WorkloadSpec, ext *extensions.H
"spec": workloadSpec,
}

var externals = map[string]interface{}{}
var externals = make(map[string]interface{})
var shared = make([]humanitec.UpdateAction, 0)
for name, res := range spec.Resources {
if meta, exists := ext.Resources[name]; !exists || meta.Scope == "" || meta.Scope == "external" {
if res.Type != "service" && res.Type != "environment" {
externals[name] = map[string]interface{}{
switch res.Type {
ghen marked this conversation as resolved.
Show resolved Hide resolved

case "service", "environment":
continue

default:
resId, hasAnnotation := res.Metadata.Annotations[AnnotationLabelResourceId]
if resId == "" {
resId = fmt.Sprintf("externals.%s", name)
}

// DEPRECATED: Should use resource annotations instead
if meta, hasMeta := ext.Resources[name]; hasMeta {
log.Printf("Warning: Extensions for resources has been deprecated. Use Score resource annotations instead. Extensions are stil configured for '%s'.\n", name)
ghen marked this conversation as resolved.
Show resolved Hide resolved
if !hasAnnotation && (meta.Scope == "" || meta.Scope == "externals") {
resId = fmt.Sprintf("externals.%s", name)
} else if !hasAnnotation && meta.Scope == "shared" {
resId = fmt.Sprintf("shared.%s", name)
}
}
// END (DEPRECATED)

if strings.HasPrefix(resId, "externals.") {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is also possible to reference resources in other workloads - e.g. modules.other-workload.externals.my-res
This case would fail here.

This does not require explicit addition of a resource to the delta, but it should be allowed.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are adding a new feature here. This logic was not supported with the extensions file mechanism.
Let's create a separate issue and describe possible cases, requirements, and app behaviour there.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This third case is as follows:
the res-id can have the format:

modules.{workloadId}.externals.{resId}

In this case, there are 2 outcomes:

  1. if {workloadId} matches the current workload, treat as externals.{resId}
  2. if {workloadId} does not match the current workload, just re-render the placeholder as having the value of the annotation (i.e. don't add anything to externals or shared properties.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

workload-id is unknown to score. what do we compare with?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we to support short versions if the same res-id? E.g. external.{resId}? What about app-shared res-ids? How would they look like?

Copy link
Contributor

@chrishumanitec chrishumanitec May 2, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

workload-id is unknown to score. what do we compare with?
The workload ID is known because the delta includes it. This is the workload ID to consider.

Here are the formats to support:

  1. shared.{resId} -> replace
  2. externals.{resId} -> treat at externals.{resId}
  3. modules.{workloadId}.externals.{redId} -> replace. If workloadId == current workload ID (as determined when creating the delta treat at externals.{resId} otherwise replace.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

var resName = strings.Replace(resId, "externals.", "", 1)
var extRes = map[string]interface{}{
"type": res.Type,
}
if len(res.Params) > 0 {
extRes["params"] = res.Params
}
externals[resName] = extRes
} else if strings.HasPrefix(resId, "shared.") {
var resName = strings.Replace(resId, "shared.", "", 1)
var sharedRes = map[string]interface{}{
"type": res.Type,
}
if len(res.Params) > 0 {
sharedRes["params"] = res.Params
}
shared = append(shared, humanitec.UpdateAction{
Operation: "add",
Path: "/" + resName,
Value: sharedRes,
})
} else {
log.Printf("Warning: Invalid resource id value '%s'. Not supported.\n", resId)
}
}
}
if len(externals) > 0 {
workload["externals"] = externals
}

var shared []humanitec.UpdateAction
for name, res := range spec.Resources {
if meta, exists := ext.Resources[name]; exists && meta.Scope == "shared" {
if shared == nil {
shared = make([]humanitec.UpdateAction, 0)
}
shared = append(shared, humanitec.UpdateAction{
Operation: "add",
Path: "/" + name,
Value: map[string]interface{}{
"type": res.Type,
},
})
}
}

var res = humanitec.CreateDeploymentDeltaRequest{
Metadata: humanitec.DeltaMetadata{
Name: name,
Expand All @@ -197,7 +228,9 @@ func ConvertSpec(name, envID string, spec *score.WorkloadSpec, ext *extensions.H
spec.Metadata.Name: workload,
},
},
Shared: shared,
}
if len(shared) > 0 {
res.Shared = shared
}

return &res, nil
Expand Down
39 changes: 37 additions & 2 deletions internal/humanitec/convert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,11 @@ func TestScoreConvert(t *testing.T) {
},
Resources: map[string]score.ResourceSpec{
"env": {
Metadata: score.ResourceMeta{
Annotations: map[string]string{
AnnotationLabelResourceId: "externals.should-ignore-this-one",
},
},
Type: "environment",
Properties: map[string]score.ResourcePropertySpec{
"DEBUG": {Default: false, Required: false},
Expand All @@ -202,11 +207,19 @@ func TestScoreConvert(t *testing.T) {
Properties: map[string]score.ResourcePropertySpec{
"domain": {},
},
Params: map[string]interface{}{
"test": "value",
},
},
"data": {
Type: "volume",
},
"db": {
Metadata: score.ResourceMeta{
Annotations: map[string]string{
AnnotationLabelResourceId: "externals.annotations-db-id",
},
},
Type: "postgres",
Properties: map[string]score.ResourcePropertySpec{
"host": {Default: "localhost", Required: true},
Expand All @@ -215,6 +228,14 @@ func TestScoreConvert(t *testing.T) {
"user_name": {Required: true, Secret: true},
"password": {Required: true, Secret: true},
},
Params: map[string]interface{}{
"extensions": map[string]interface{}{
"uuid-ossp": map[string]interface{}{
"schema": "uuid_schema",
"version": "1.1",
},
},
},
},
"orders": {
Type: "service",
Expand Down Expand Up @@ -245,6 +266,9 @@ func TestScoreConvert(t *testing.T) {
},
},
Resources: extensions.HumanitecResourcesSpecs{
"db": extensions.HumanitecResourceSpec{
Scope: "shared",
},
"dns": extensions.HumanitecResourceSpec{
Scope: "shared",
},
Expand All @@ -264,7 +288,7 @@ func TestScoreConvert(t *testing.T) {
"DEBUG": "${values.DEBUG}",
"LOGS_LEVEL": "${pod.debug.level}",
"ORDERS_SERVICE": "http://${modules.orders.service.name}:${modules.orders.service.port}/api",
"CONNECTION_STRING": "postgresql://${externals.db.host}:${externals.db.port}/${externals.db.name}",
"CONNECTION_STRING": "postgresql://${externals.annotations-db-id.host}:${externals.annotations-db-id.port}/${externals.annotations-db-id.name}",
"DOMAIN_NAME": "${shared.dns.domain}",
},
"files": map[string]interface{}{
Expand Down Expand Up @@ -302,8 +326,16 @@ func TestScoreConvert(t *testing.T) {
"data": map[string]interface{}{
"type": "volume",
},
"db": map[string]interface{}{
"annotations-db-id": map[string]interface{}{
"type": "postgres",
"params": map[string]interface{}{
"extensions": map[string]interface{}{
"uuid-ossp": map[string]interface{}{
"schema": "uuid_schema",
"version": "1.1",
},
},
},
},
},
},
Expand All @@ -315,6 +347,9 @@ func TestScoreConvert(t *testing.T) {
Path: "/dns",
Value: map[string]interface{}{
"type": "dns",
"params": map[string]interface{}{
"test": "value",
},
},
},
},
Expand Down
10 changes: 6 additions & 4 deletions internal/humanitec/extensions/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ package extensions
// dns:
// scope: shared
type HumanitecExtensionsSpec struct {
ApiVersion string `mapstructure:"apiVersion"`
Profile string `mapstructure:"profile"`
Spec map[string]interface{} `mapstructure:"spec"`
Resources HumanitecResourcesSpecs `mapstructure:"resources"`
ApiVersion string `mapstructure:"apiVersion"`
Profile string `mapstructure:"profile"`
Spec map[string]interface{} `mapstructure:"spec"`

// DEPRECATED: Should use score resources annotations instead
Resources HumanitecResourcesSpecs `mapstructure:"resources"`
}

// HumanitecResourcesSpecs is a map of workload resources specifications.
Expand Down
15 changes: 13 additions & 2 deletions internal/humanitec/templates.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,19 @@ func buildContext(metadata score.WorkloadMeta, resources score.ResourcesSpecs, e
if res.Type == "workload" {
log.Println("Warning: 'workload' is a reserved resource type. Its usage may lead to compatibility issues with future releases of this application.")
}
if resExt, exists := ext[resName]; exists && resExt.Scope == "shared" {
source = fmt.Sprintf("shared.%s", resName)
resId, hasAnnotation := res.Metadata.Annotations[AnnotationLabelResourceId]
// DEPRECATED: Should use resource annotations instead
if resExt, hasMeta := ext[resName]; hasMeta && !hasAnnotation {
if resExt.Scope == "" || resExt.Scope == "external" {
resId = fmt.Sprintf("externals.%s", resName)
} else if resExt.Scope == "shared" {
resId = fmt.Sprintf("shared.%s", resName)
}
}
// END (DEPRECATED)

if resId != "" {
source = resId
} else {
source = fmt.Sprintf("externals.%s", resName)
}
Expand Down