diff --git a/go.mod b/go.mod index f468556ec3..b470226b7a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/clarketm/json v1.14.1 github.com/containers/image/v5 v5.17.0 github.com/containers/storage v1.37.0 - github.com/coreos/fcct v0.5.0 + github.com/coreos/butane v0.14.0 github.com/coreos/go-semver v0.3.0 github.com/coreos/ign-converter v0.0.0-20201123214124-8dac862888aa github.com/coreos/ignition v0.35.0 @@ -29,10 +29,9 @@ require ( github.com/openshift/client-go v0.0.0-20220411091747-c100636b0bc0 github.com/openshift/library-go v0.0.0-20211220195323-eca2c467c492 github.com/openshift/runtime-utils v0.0.0-20220225175100-8dec0d84fb39 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 github.com/spf13/cobra v1.2.1 - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace github.com/stretchr/testify v1.7.0 github.com/vincent-petithory/dataurl v1.0.0 golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b @@ -49,6 +48,11 @@ require ( sigs.k8s.io/controller-runtime v0.11.0 ) +require ( + github.com/coreos/go-json v0.0.0-20211020211907-c63f628265de // indirect + github.com/coreos/vcontext v0.0.0-20211021162308-f1dbbca7bef4 // indirect +) + require ( 4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect github.com/Antonboom/errname v0.1.4 // indirect @@ -76,10 +80,8 @@ require ( github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b // indirect github.com/containers/ocicrypt v1.1.2 // indirect - github.com/coreos/go-json v0.0.0-20211020211907-c63f628265de // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/coreos/vcontext v0.0.0-20211021162308-f1dbbca7bef4 // indirect github.com/daixiang0/gci v0.2.9 // indirect github.com/denis-tingajkin/go-header v0.4.2 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect @@ -193,6 +195,7 @@ require ( github.com/pelletier/go-toml v1.9.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349 // indirect github.com/prometheus/client_model v0.2.0 // indirect diff --git a/go.sum b/go.sum index db7877a0fd..fc5f19cfb8 100644 --- a/go.sum +++ b/go.sum @@ -163,7 +163,6 @@ github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c h1:icme0QhxrgZOx github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c/go.mod h1:BRljTyotlu+6N+Qlu5MhjxpdmccCnp9lDvZjNNV8qr4= github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.28/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -334,10 +333,10 @@ github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= github.com/coredns/corefile-migration v1.0.14/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/butane v0.14.0 h1:1xLt5y6RR8NTmeDf6yMzqP7Jqre8PvJ/1BTXQTxEMhk= +github.com/coreos/butane v0.14.0/go.mod h1:Q5DcBsHDckEZ7IgQSb1MvvkNc50dpoT1lOHdGWwCRjY= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/fcct v0.5.0 h1:f/z+MCoR2vULes+MyoPEApQ6iluy/JbXoRi6dahPItQ= -github.com/coreos/fcct v0.5.0/go.mod h1:cbE+j77YSQwFB2fozWVB3qsI2Pi3YiVEbDz/b6Yywdo= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -362,14 +361,12 @@ github.com/coreos/ign-converter v0.0.0-20201123214124-8dac862888aa h1:oIF6XCee+G github.com/coreos/ign-converter v0.0.0-20201123214124-8dac862888aa/go.mod h1:pqAsDWa5YDi10Va/aqQI0bwOs9hXqoE2xwb5vnFys5s= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= -github.com/coreos/ignition/v2 v2.1.1/go.mod h1:RqmqU64zxarUJa3l4cHtbhcSwfQLpUhv0WVziZwoXvE= github.com/coreos/ignition/v2 v2.7.0/go.mod h1:3CjaRpg51hmJzPjarbzB0RvSZbLkNOczxKJobTl6nOY= github.com/coreos/ignition/v2 v2.13.0 h1:1ouW+d0nOuPUbLjxxOCnC+dWQxynr8Mt5exqJoCD7b4= github.com/coreos/ignition/v2 v2.13.0/go.mod h1:HO1HWYWcvAIbHu6xewoKxPGBTyZ32FLwGIuipw5d63o= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= -github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20211021162308-f1dbbca7bef4 h1:pfSsrvbjUFGINaPGy0mm2QKQKTdq7IcbUa+nQwsz2UM= github.com/coreos/vcontext v0.0.0-20211021162308-f1dbbca7bef4/go.mod h1:HckqHnP/HI41vS0bfVjJ20u6jD0biI5+68QwZm5Xb9U= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -1233,8 +1230,9 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= @@ -1494,7 +1492,6 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1613,7 +1610,6 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2006,7 +2002,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index f2640dd815..96a8c63cd3 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -16,7 +16,9 @@ import ( "strings" "github.com/clarketm/json" - fcctbase "github.com/coreos/fcct/base/v0_1" + bubase "github.com/coreos/butane/base/v0_3" + bucommon "github.com/coreos/butane/config/common" + fcosbutane "github.com/coreos/butane/config/fcos/v1_2" "github.com/coreos/ign-converter/translate/v23tov30" "github.com/coreos/ign-converter/translate/v32tov22" "github.com/coreos/ign-converter/translate/v32tov31" @@ -600,47 +602,54 @@ func removeIgnDuplicateFilesUnitsUsers(ignConfig ign2types.Config) (ign2types.Co return ignConfig, nil } -// TranspileCoreOSConfigToIgn transpiles Fedora CoreOS config to ignition -// internally it transpiles to Ign spec v3 config -func TranspileCoreOSConfigToIgn(files, units []string) (*ign3types.Config, error) { +// ConvertButaneFragmentsToIgnition takes as input butane fragments (fcos 1.3) and returns an Ignition v3.2 configuration. +func ConvertButaneFragmentsToIgnition(files, units []string) (*ign3types.Config, error) { overwrite := true - outConfig := ign3types.Config{} + buCfg := fcosbutane.Config{} // Convert data to Ignition resources for _, contents := range files { - f := new(fcctbase.File) + f := new(bubase.File) if err := yaml.Unmarshal([]byte(contents), f); err != nil { return nil, fmt.Errorf("failed to unmarshal %q into struct: %w", contents, err) } f.Overwrite = &overwrite - // Add the file to the config - var ctCfg fcctbase.Config - ctCfg.Storage.Files = append(ctCfg.Storage.Files, *f) - ign3_0config, tSet, err := ctCfg.ToIgn3_0() - if err != nil { - return nil, fmt.Errorf("failed to transpile config to Ignition config %w\nTranslation set: %v", err, tSet) - } - ign3_2config := translate3.Translate(translate3_1.Translate(ign3_0config)) - outConfig = ign3.Merge(outConfig, ign3_2config) + buCfg.Storage.Files = append(buCfg.Storage.Files, *f) } + dropins := make(map[string]bubase.Unit) for _, contents := range units { - u := new(fcctbase.Unit) + u := new(bubase.Unit) if err := yaml.Unmarshal([]byte(contents), u); err != nil { return nil, fmt.Errorf("failed to unmarshal systemd unit into struct: %w", err) } - // Add the unit to the config - var ctCfg fcctbase.Config - ctCfg.Systemd.Units = append(ctCfg.Systemd.Units, *u) - ign3_0config, tSet, err := ctCfg.ToIgn3_0() - if err != nil { - return nil, fmt.Errorf("failed to transpile config to Ignition config %w\nTranslation set: %v", err, tSet) + // Handle drop-ins specially; a unit name with drop-ins can be repeated and they should be merged. + if len(u.Dropins) > 0 { + if current, ok := dropins[u.Name]; ok { + current.Dropins = append(current.Dropins, u.Dropins...) + } else { + dropins[u.Name] = *u + } + } else { + buCfg.Systemd.Units = append(buCfg.Systemd.Units, *u) } - ign3_2config := translate3.Translate(translate3_1.Translate(ign3_0config)) - outConfig = ign3.Merge(outConfig, ign3_2config) + } + for _, unit := range dropins { + buCfg.Systemd.Units = append(buCfg.Systemd.Units, unit) + } + + // We want the output of this to be reproducible (it should not depend on the order of the filesystem) + sort.Slice(buCfg.Storage.Files, func(i, j int) bool { return buCfg.Storage.Files[i].Path < buCfg.Storage.Files[j].Path }) + sort.Slice(buCfg.Systemd.Units, func(i, j int) bool { return buCfg.Systemd.Units[i].Name < buCfg.Systemd.Units[j].Name }) + for _, unit := range buCfg.Systemd.Units { + sort.Slice(unit.Dropins, func(i, j int) bool { return unit.Dropins[i].Name < unit.Dropins[j].Name }) } + outConfig, rep, err := buCfg.ToIgn3_2(bucommon.TranslateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to convert butane fragments: %v %w", rep, err) + } return &outConfig, nil } diff --git a/pkg/controller/common/helpers_test.go b/pkg/controller/common/helpers_test.go index 087dfecf27..0f7ea93821 100644 --- a/pkg/controller/common/helpers_test.go +++ b/pkg/controller/common/helpers_test.go @@ -18,7 +18,15 @@ import ( "github.com/openshift/machine-config-operator/test/helpers" ) -func TestTranspileCoreOSConfig(t *testing.T) { +func TestConvertButaneFragmentsToIgnition(t *testing.T) { + // Test the null case + config, err := ConvertButaneFragmentsToIgnition([]string{}, []string{}) + require.NoError(t, err) + if report := validate3.ValidateWithContext(config, nil); report.IsFatal() { + t.Fatalf("invalid ignition V3 config found: %v", report) + } + require.Equal(t, len(config.Storage.Files), 0) + kubeletConfig := ` mode: 0644 path: "/etc/kubernetes/kubelet.conf" @@ -58,13 +66,13 @@ dropins: [Unit] ConditionPathExists=/enoent ` - config, err := TranspileCoreOSConfigToIgn([]string{kubeletConfig, auditConfig}, []string{kubeletService, crioDropin, dockerDropin}) + config, err = ConvertButaneFragmentsToIgnition([]string{kubeletConfig, auditConfig}, []string{kubeletService, crioDropin, dockerDropin}) require.Nil(t, err) if report := validate3.ValidateWithContext(config, nil); report.IsFatal() { t.Fatalf("invalid ignition V3 config found: %v", report) } require.Equal(t, len(config.Storage.Files), 2) - require.True(t, strings.HasPrefix(*config.Storage.Files[0].Contents.Source, "data:,kind%3A%20KubeletConfiguration%0Aapi")) + require.True(t, strings.HasPrefix(*config.Storage.Files[1].Contents.Source, "data:,kind%3A%20KubeletConfiguration%0Aapi")) require.Equal(t, len(config.Systemd.Units), 3) } diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 00015549a4..31949b784f 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -285,9 +285,9 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, return vs } - ignCfg, err := ctrlcommon.TranspileCoreOSConfigToIgn(keySortVals(files), keySortVals(units)) + ignCfg, err := ctrlcommon.ConvertButaneFragmentsToIgnition(keySortVals(files), keySortVals(units)) if err != nil { - return nil, fmt.Errorf("error transpiling CoreOS config to Ignition config: %w", err) + return nil, fmt.Errorf("error converting Butane fragments to Ignition config: %w", err) } mcfg, err := ctrlcommon.MachineConfigFromIgnConfig(role, name, ignCfg) if err != nil { diff --git a/vendor/github.com/coreos/fcct/LICENSE b/vendor/github.com/coreos/butane/LICENSE similarity index 100% rename from vendor/github.com/coreos/fcct/LICENSE rename to vendor/github.com/coreos/butane/LICENSE diff --git a/vendor/github.com/coreos/butane/base/util/file.go b/vendor/github.com/coreos/butane/base/util/file.go new file mode 100644 index 0000000000..a6f9168088 --- /dev/null +++ b/vendor/github.com/coreos/butane/base/util/file.go @@ -0,0 +1,146 @@ +// Copyright 2020 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package util + +import ( + "path/filepath" + "strings" + + "github.com/coreos/butane/config/common" +) + +func EnsurePathWithinFilesDir(path, filesDir string) error { + absBase, err := filepath.Abs(filesDir) + if err != nil { + return err + } + absPath, err := filepath.Abs(path) + if err != nil { + return err + } + if !strings.HasPrefix(absPath, absBase+string(filepath.Separator)) { + return common.ErrFilesDirEscape + } + return nil +} + +/// CheckForDecimalMode fails if the specified mode appears to have been +/// incorrectly specified in decimal instead of octal. +func CheckForDecimalMode(mode int, directory bool) error { + correctedMode, ok := decimalModeToOctal(mode) + if !ok { + return nil + } + if !isTypicalMode(mode, directory) && isTypicalMode(correctedMode, directory) { + return common.ErrDecimalMode + } + return nil +} + +/// isTypicalMode returns true if the specified mode is unsurprising. +/// It returns false for some modes that are unusual but valid in limited +/// cases. +func isTypicalMode(mode int, directory bool) bool { + // no permissions is always reasonable (root ignores mode bits) + if mode == 0 { + return true + } + + // test user/group/other in reverse order + perms := []int{mode & 0007, (mode & 0070) >> 3, (mode & 0700) >> 6} + hadR := false + hadW := false + hadX := false + for _, perm := range perms { + r := perm&4 != 0 + w := perm&2 != 0 + x := perm&1 != 0 + // more-specific perm must have all the bits of less-specific + // perm (r--rw----) + if !r && hadR || !w && hadW || !x && hadX { + return false + } + // if we have executable permission, it's weird for a + // less-specific perm to have read but not execute (rwxr-----) + if x && hadR && !hadX { + return false + } + // -w- and --x are reasonable in special cases but they're + // uncommon + if (w || x) && !r { + return false + } + hadR = hadR || r + hadW = hadW || w + hadX = hadX || x + } + + // must be readable by someone + if !hadR { + return false + } + + if directory { + // must be executable by someone + if !hadX { + return false + } + // setuid forbidden + if mode&04000 != 0 { + return false + } + // setgid or sticky must be writable to someone + if mode&03000 != 0 && !hadW { + return false + } + } else { + // setuid or setgid + if mode&06000 != 0 { + // must be executable to someone + if !hadX { + return false + } + // world-writable permission is a bad idea + if mode&2 != 0 { + return false + } + } + // sticky forbidden + if mode&01000 != 0 { + return false + } + } + + return true +} + +/// decimalModeToOctal takes a mode written in decimal and converts it to +/// octal, returning (0, false) on failure. +func decimalModeToOctal(mode int) (int, bool) { + if mode < 0 || mode > 7777 { + // out of range + return 0, false + } + ret := 0 + for divisor := 1000; divisor > 0; divisor /= 10 { + digit := (mode / divisor) % 10 + if digit > 7 { + // digit not available in octal + return 0, false + } + ret = (ret << 3) | digit + } + return ret, true +} diff --git a/vendor/github.com/coreos/butane/base/util/merge.go b/vendor/github.com/coreos/butane/base/util/merge.go new file mode 100644 index 0000000000..ad8c3b875e --- /dev/null +++ b/vendor/github.com/coreos/butane/base/util/merge.go @@ -0,0 +1,75 @@ +// Copyright 2020 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package util + +import ( + "fmt" + + "github.com/coreos/butane/translate" + + "github.com/coreos/ignition/v2/config/merge" +) + +// MergeTranslatedConfigs merges a parent and child config and returns the +// result. It also generates and returns the merged TranslationSet by +// mapping the parent/child TranslationSets through the merge transcript. +func MergeTranslatedConfigs(parent interface{}, parentTranslations translate.TranslationSet, child interface{}, childTranslations translate.TranslationSet) (interface{}, translate.TranslationSet) { + // mappings: + // left: parent or child translate.TranslationSet + // right: merge.Transcript + + // merge configs + result, right := merge.MergeStructTranscribe(parent, child) + + // merge left and right mappings into new TranslationSet + if parentTranslations.FromTag != childTranslations.FromTag || parentTranslations.ToTag != childTranslations.ToTag { + panic(fmt.Sprintf("mismatched translation tags, %s != %s || %s != %s", parentTranslations.FromTag, childTranslations.FromTag, parentTranslations.ToTag, childTranslations.ToTag)) + } + ts := translate.NewTranslationSet(parentTranslations.FromTag, parentTranslations.ToTag) + for _, rightEntry := range right.Mappings { + var left *translate.TranslationSet + switch rightEntry.From.Tag { + case merge.TAG_PARENT: + left = &parentTranslations + case merge.TAG_CHILD: + left = &childTranslations + default: + panic("unexpected mapping tag " + rightEntry.From.Tag) + } + leftEntry, ok := left.Set[rightEntry.From.String()] + if !ok { + // the right mapping is more comprehensive than the + // left mapping + continue + } + if _, ok := ts.Set[rightEntry.To.String()]; ok && rightEntry.From.Tag != merge.TAG_CHILD { + // For result fields which are produced by combining + // the parent and child, there will be two + // transcript entries, one for each side. We want + // to prefer the child because the parent is + // probably a desugared config whose source is + // textually unrelated to the result config. + // + // Currently, Ignition always reports parent before + // child, but that isn't necessarily contractual, so + // we don't assume it. Here, we've found the second + // entry and it's not from the child; skip it. + continue + } + rightEntry.To.Tag = leftEntry.To.Tag + ts.AddTranslation(leftEntry.From, rightEntry.To) + } + return result, ts +} diff --git a/vendor/github.com/coreos/butane/base/util/test.go b/vendor/github.com/coreos/butane/base/util/test.go new file mode 100644 index 0000000000..f4d8ca83f5 --- /dev/null +++ b/vendor/github.com/coreos/butane/base/util/test.go @@ -0,0 +1,43 @@ +// Copyright 2020 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package util + +import ( + "testing" + + "github.com/coreos/butane/translate" + "github.com/stretchr/testify/assert" +) + +// helper functions for writing tests + +// VerifyTranslations ensures all the translations are identity, unless they +// match a listed one, and verifies that all the listed ones exist. +func VerifyTranslations(t *testing.T, set translate.TranslationSet, exceptions []translate.Translation) { + exceptionSet := translate.NewTranslationSet(set.FromTag, set.ToTag) + for _, ex := range exceptions { + exceptionSet.AddTranslation(ex.From, ex.To) + if tr, ok := set.Set[ex.To.String()]; ok { + assert.Equal(t, ex, tr, "non-identity translation with unexpected From") + } else { + t.Errorf("missing non-identity translation %v", ex) + } + } + for key, translation := range set.Set { + if _, ok := exceptionSet.Set[key]; !ok { + assert.Equal(t, translation.From.Path, translation.To.Path, "translation is not identity") + } + } +} diff --git a/vendor/github.com/coreos/butane/base/util/url.go b/vendor/github.com/coreos/butane/base/util/url.go new file mode 100644 index 0000000000..2cf3bef070 --- /dev/null +++ b/vendor/github.com/coreos/butane/base/util/url.go @@ -0,0 +1,68 @@ +// Copyright 2020 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package util + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "net/url" + + "github.com/coreos/ignition/v2/config/util" + "github.com/vincent-petithory/dataurl" +) + +func MakeDataURL(contents []byte, currentCompression *string, allowCompression bool) (uri string, gzipped bool, err error) { + // try three different encodings, and select the smallest one + + // URL-escaped, useful for ASCII text + opaque := "," + dataurl.Escape(contents) + + // Base64-encoded, useful for small or incompressible binary data + b64 := ";base64," + base64.StdEncoding.EncodeToString(contents) + if len(b64) < len(opaque) { + opaque = b64 + } + + // Base64-encoded gzipped, useful for compressible data. If the + // user already enabled compression, don't compress again. + // We don't try base64-encoded URL-escaped because gzipped data is + // binary and URL escaping is unlikely to be efficient. + if util.NilOrEmpty(currentCompression) && allowCompression { + var buf bytes.Buffer + var compressor *gzip.Writer + if compressor, err = gzip.NewWriterLevel(&buf, gzip.BestCompression); err != nil { + return + } + if _, err = compressor.Write(contents); err != nil { + return + } + if err = compressor.Close(); err != nil { + return + } + gz := ";base64," + base64.StdEncoding.EncodeToString(buf.Bytes()) + // Account for space needed by "compression": "gzip". + if len(gz)+25 < len(opaque) { + opaque = gz + gzipped = true + } + } + + uri = (&url.URL{ + Scheme: "data", + Opaque: opaque, + }).String() + return +} diff --git a/vendor/github.com/coreos/fcct/base/v0_1/schema.go b/vendor/github.com/coreos/butane/base/v0_3/schema.go similarity index 64% rename from vendor/github.com/coreos/fcct/base/v0_1/schema.go rename to vendor/github.com/coreos/butane/base/v0_3/schema.go index c1a70bcb98..2b91822557 100644 --- a/vendor/github.com/coreos/fcct/base/v0_1/schema.go +++ b/vendor/github.com/coreos/butane/base/v0_3/schema.go @@ -12,23 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License.) -package v0_1 +package v0_3 -type CaReference struct { - Source string `yaml:"source"` - Verification Verification `yaml:"verification"` +type Clevis struct { + Custom *Custom `yaml:"custom"` + Tang []Tang `yaml:"tang"` + Threshold *int `yaml:"threshold"` + Tpm2 *bool `yaml:"tpm2"` } type Config struct { + Version string `yaml:"version"` + Variant string `yaml:"variant"` Ignition Ignition `yaml:"ignition"` Passwd Passwd `yaml:"passwd"` Storage Storage `yaml:"storage"` Systemd Systemd `yaml:"systemd"` } -type ConfigReference struct { - Source *string `yaml:"source"` - Verification Verification `yaml:"verification"` +type Custom struct { + Config string `yaml:"config"` + NeedsNetwork *bool `yaml:"needs_network"` + Pin string `yaml:"pin"` } type Device string @@ -53,45 +58,48 @@ type Dropin struct { } type File struct { - Group NodeGroup `yaml:"group"` - Overwrite *bool `yaml:"overwrite"` - Path string `yaml:"path"` - User NodeUser `yaml:"user"` - Append []FileContents `yaml:"append"` - Contents FileContents `yaml:"contents"` - Mode *int `yaml:"mode"` -} - -type FileContents struct { - Compression *string `yaml:"compression"` - Source *string `yaml:"source"` - Inline *string `yaml:"inline"` // Added, not in ignition spec - Verification Verification `yaml:"verification"` + Group NodeGroup `yaml:"group"` + Overwrite *bool `yaml:"overwrite"` + Path string `yaml:"path"` + User NodeUser `yaml:"user"` + Append []Resource `yaml:"append"` + Contents Resource `yaml:"contents"` + Mode *int `yaml:"mode"` } type Filesystem struct { - Device string `yaml:"device"` - Format *string `yaml:"format"` - Label *string `yaml:"label"` - Options []FilesystemOption `yaml:"options"` - Path *string `yaml:"path"` - UUID *string `yaml:"uuid"` - WipeFilesystem *bool `yaml:"wipe_filesystem"` + Device string `yaml:"device"` + Format *string `yaml:"format"` + Label *string `yaml:"label"` + MountOptions []string `yaml:"mount_options"` + Options []string `yaml:"options"` + Path *string `yaml:"path"` + UUID *string `yaml:"uuid"` + WipeFilesystem *bool `yaml:"wipe_filesystem"` + WithMountUnit *bool `yaml:"with_mount_unit" butane:"auto_skip"` // Added, not in Ignition spec } type FilesystemOption string type Group string +type HTTPHeader struct { + Name string `yaml:"name"` + Value *string `yaml:"value"` +} + +type HTTPHeaders []HTTPHeader + type Ignition struct { Config IgnitionConfig `yaml:"config"` + Proxy Proxy `yaml:"proxy"` Security Security `yaml:"security"` Timeouts Timeouts `yaml:"timeouts"` } type IgnitionConfig struct { - Merge []ConfigReference `yaml:"merge"` - Replace ConfigReference `yaml:"replace"` + Merge []Resource `yaml:"merge"` + Replace Resource `yaml:"replace"` } type Link struct { @@ -103,6 +111,19 @@ type Link struct { Target string `yaml:"target"` } +type Luks struct { + Clevis *Clevis `yaml:"clevis"` + Device *string `yaml:"device"` + KeyFile Resource `yaml:"key_file"` + Label *string `yaml:"label"` + Name string `yaml:"name"` + Options []LuksOption `yaml:"options"` + UUID *string `yaml:"uuid"` + WipeVolume *bool `yaml:"wipe_volume"` +} + +type LuksOption string + type NodeGroup struct { ID *int `yaml:"id"` Name *string `yaml:"name"` @@ -117,6 +138,7 @@ type Partition struct { GUID *string `yaml:"guid"` Label *string `yaml:"label"` Number int `yaml:"number"` + Resize *bool `yaml:"resize"` ShouldExist *bool `yaml:"should_exist"` SizeMiB *int `yaml:"size_mib"` StartMiB *int `yaml:"start_mib"` @@ -133,6 +155,7 @@ type PasswdGroup struct { Gid *int `yaml:"gid"` Name string `yaml:"name"` PasswordHash *string `yaml:"password_hash"` + ShouldExist *bool `yaml:"should_exist"` System *bool `yaml:"system"` } @@ -146,12 +169,19 @@ type PasswdUser struct { NoUserGroup *bool `yaml:"no_user_group"` PasswordHash *string `yaml:"password_hash"` PrimaryGroup *string `yaml:"primary_group"` + ShouldExist *bool `yaml:"should_exist"` SSHAuthorizedKeys []SSHAuthorizedKey `yaml:"ssh_authorized_keys"` Shell *string `yaml:"shell"` System *bool `yaml:"system"` UID *int `yaml:"uid"` } +type Proxy struct { + HTTPProxy *string `yaml:"http_proxy"` + HTTPSProxy *string `yaml:"https_proxy"` + NoProxy []string `yaml:"no_proxy"` +} + type Raid struct { Devices []Device `yaml:"devices"` Level string `yaml:"level"` @@ -162,6 +192,15 @@ type Raid struct { type RaidOption string +type Resource struct { + Compression *string `yaml:"compression"` + HTTPHeaders HTTPHeaders `yaml:"http_headers"` + Source *string `yaml:"source"` + Inline *string `yaml:"inline"` // Added, not in ignition spec + Local *string `yaml:"local"` // Added, not in ignition spec + Verification Verification `yaml:"verification"` +} + type SSHAuthorizedKey string type Security struct { @@ -174,15 +213,22 @@ type Storage struct { Files []File `yaml:"files"` Filesystems []Filesystem `yaml:"filesystems"` Links []Link `yaml:"links"` + Luks []Luks `yaml:"luks"` Raid []Raid `yaml:"raid"` + Trees []Tree `yaml:"trees" butane:"auto_skip"` // Added, not in ignition spec } type Systemd struct { Units []Unit `yaml:"units"` } +type Tang struct { + Thumbprint *string `yaml:"thumbprint"` + URL string `yaml:"url"` +} + type TLS struct { - CertificateAuthorities []CaReference `yaml:"certificate_authorities"` + CertificateAuthorities []Resource `yaml:"certificate_authorities"` } type Timeouts struct { @@ -190,6 +236,11 @@ type Timeouts struct { HTTPTotal *int `yaml:"http_total"` } +type Tree struct { + Local string `yaml:"local"` + Path *string `yaml:"path"` +} + type Unit struct { Contents *string `yaml:"contents"` Dropins []Dropin `yaml:"dropins"` diff --git a/vendor/github.com/coreos/butane/base/v0_3/translate.go b/vendor/github.com/coreos/butane/base/v0_3/translate.go new file mode 100644 index 0000000000..cf8763423f --- /dev/null +++ b/vendor/github.com/coreos/butane/base/v0_3/translate.go @@ -0,0 +1,407 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_3 + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "text/template" + + baseutil "github.com/coreos/butane/base/util" + "github.com/coreos/butane/config/common" + "github.com/coreos/butane/translate" + + "github.com/coreos/go-systemd/unit" + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/ignition/v2/config/v3_2/types" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +var ( + mountUnitTemplate = template.Must(template.New("unit").Parse(`# Generated by Butane +[Unit] +{{- if .Remote }} +Before=remote-fs.target +DefaultDependencies=no +{{- else }} +Before=local-fs.target +{{- end }} +Requires=systemd-fsck@{{.EscapedDevice}}.service +After=systemd-fsck@{{.EscapedDevice}}.service + +[Mount] +Where={{.Path}} +What={{.Device}} +Type={{.Format}} +{{- if .MountOptions }} +Options= + {{- range $i, $opt := .MountOptions }} + {{- if $i }},{{ end }} + {{- $opt }} + {{- end }} +{{- end }} + +[Install] +{{- if .Remote }} +RequiredBy=remote-fs.target +{{- else }} +RequiredBy=local-fs.target +{{- end }}`)) +) + +// ToIgn3_2Unvalidated translates the config to an Ignition config. It also returns the set of translations +// it did so paths in the resultant config can be tracked back to their source in the source config. +// No config validation is performed on input or output. +func (c Config) ToIgn3_2Unvalidated(options common.TranslateOptions) (types.Config, translate.TranslationSet, report.Report) { + ret := types.Config{} + + tr := translate.NewTranslator("yaml", "json", options) + tr.AddCustomTranslator(translateIgnition) + tr.AddCustomTranslator(translateFile) + tr.AddCustomTranslator(translateDirectory) + tr.AddCustomTranslator(translateLink) + tr.AddCustomTranslator(translateResource) + + tm, r := translate.Prefixed(tr, "ignition", &c.Ignition, &ret.Ignition) + tm.AddTranslation(path.New("yaml", "version"), path.New("json", "ignition", "version")) + tm.AddTranslation(path.New("yaml", "ignition"), path.New("json", "ignition")) + translate.MergeP(tr, tm, &r, "passwd", &c.Passwd, &ret.Passwd) + translate.MergeP(tr, tm, &r, "storage", &c.Storage, &ret.Storage) + translate.MergeP(tr, tm, &r, "systemd", &c.Systemd, &ret.Systemd) + + c.addMountUnits(&ret, &tm) + + tm2, r2 := c.processTrees(&ret, options) + tm.Merge(tm2) + r.Merge(r2) + + if r.IsFatal() { + return types.Config{}, translate.TranslationSet{}, r + } + return ret, tm, r +} + +func translateIgnition(from Ignition, options common.TranslateOptions) (to types.Ignition, tm translate.TranslationSet, r report.Report) { + tr := translate.NewTranslator("yaml", "json", options) + tr.AddCustomTranslator(translateResource) + to.Version = types.MaxVersion.String() + tm, r = translate.Prefixed(tr, "config", &from.Config, &to.Config) + translate.MergeP(tr, tm, &r, "proxy", &from.Proxy, &to.Proxy) + translate.MergeP(tr, tm, &r, "security", &from.Security, &to.Security) + translate.MergeP(tr, tm, &r, "timeouts", &from.Timeouts, &to.Timeouts) + return +} + +func translateFile(from File, options common.TranslateOptions) (to types.File, tm translate.TranslationSet, r report.Report) { + tr := translate.NewTranslator("yaml", "json", options) + tr.AddCustomTranslator(translateResource) + tm, r = translate.Prefixed(tr, "group", &from.Group, &to.Group) + translate.MergeP(tr, tm, &r, "user", &from.User, &to.User) + translate.MergeP(tr, tm, &r, "append", &from.Append, &to.Append) + translate.MergeP(tr, tm, &r, "contents", &from.Contents, &to.Contents) + translate.MergeP(tr, tm, &r, "overwrite", &from.Overwrite, &to.Overwrite) + translate.MergeP(tr, tm, &r, "path", &from.Path, &to.Path) + translate.MergeP(tr, tm, &r, "mode", &from.Mode, &to.Mode) + return +} + +func translateResource(from Resource, options common.TranslateOptions) (to types.Resource, tm translate.TranslationSet, r report.Report) { + tr := translate.NewTranslator("yaml", "json", options) + tm, r = translate.Prefixed(tr, "verification", &from.Verification, &to.Verification) + translate.MergeP(tr, tm, &r, "httpHeaders", &from.HTTPHeaders, &to.HTTPHeaders) + translate.MergeP(tr, tm, &r, "source", &from.Source, &to.Source) + translate.MergeP(tr, tm, &r, "compression", &from.Compression, &to.Compression) + + if from.Local != nil { + c := path.New("yaml", "local") + + if options.FilesDir == "" { + r.AddOnError(c, common.ErrNoFilesDir) + return + } + + // calculate file path within FilesDir and check for + // path traversal + filePath := filepath.Join(options.FilesDir, *from.Local) + if err := baseutil.EnsurePathWithinFilesDir(filePath, options.FilesDir); err != nil { + r.AddOnError(c, err) + return + } + + contents, err := ioutil.ReadFile(filePath) + if err != nil { + r.AddOnError(c, err) + return + } + + src, gzipped, err := baseutil.MakeDataURL(contents, to.Compression, !options.NoResourceAutoCompression) + if err != nil { + r.AddOnError(c, err) + return + } + to.Source = &src + tm.AddTranslation(c, path.New("json", "source")) + if gzipped { + to.Compression = util.StrToPtr("gzip") + tm.AddTranslation(c, path.New("json", "compression")) + } + } + + if from.Inline != nil { + c := path.New("yaml", "inline") + + src, gzipped, err := baseutil.MakeDataURL([]byte(*from.Inline), to.Compression, !options.NoResourceAutoCompression) + if err != nil { + r.AddOnError(c, err) + return + } + to.Source = &src + tm.AddTranslation(c, path.New("json", "source")) + if gzipped { + to.Compression = util.StrToPtr("gzip") + tm.AddTranslation(c, path.New("json", "compression")) + } + } + return +} + +func translateDirectory(from Directory, options common.TranslateOptions) (to types.Directory, tm translate.TranslationSet, r report.Report) { + tr := translate.NewTranslator("yaml", "json", options) + tm, r = translate.Prefixed(tr, "group", &from.Group, &to.Group) + translate.MergeP(tr, tm, &r, "user", &from.User, &to.User) + translate.MergeP(tr, tm, &r, "overwrite", &from.Overwrite, &to.Overwrite) + translate.MergeP(tr, tm, &r, "path", &from.Path, &to.Path) + translate.MergeP(tr, tm, &r, "mode", &from.Mode, &to.Mode) + return +} + +func translateLink(from Link, options common.TranslateOptions) (to types.Link, tm translate.TranslationSet, r report.Report) { + tr := translate.NewTranslator("yaml", "json", options) + tm, r = translate.Prefixed(tr, "group", &from.Group, &to.Group) + translate.MergeP(tr, tm, &r, "user", &from.User, &to.User) + translate.MergeP(tr, tm, &r, "target", &from.Target, &to.Target) + translate.MergeP(tr, tm, &r, "hard", &from.Hard, &to.Hard) + translate.MergeP(tr, tm, &r, "overwrite", &from.Overwrite, &to.Overwrite) + translate.MergeP(tr, tm, &r, "path", &from.Path, &to.Path) + return +} + +func (c Config) processTrees(ret *types.Config, options common.TranslateOptions) (translate.TranslationSet, report.Report) { + ts := translate.NewTranslationSet("yaml", "json") + var r report.Report + if len(c.Storage.Trees) == 0 { + return ts, r + } + t := newNodeTracker(ret) + + for i, tree := range c.Storage.Trees { + yamlPath := path.New("yaml", "storage", "trees", i) + if options.FilesDir == "" { + r.AddOnError(yamlPath, common.ErrNoFilesDir) + return ts, r + } + + // calculate base path within FilesDir and check for + // path traversal + srcBaseDir := filepath.Join(options.FilesDir, tree.Local) + if err := baseutil.EnsurePathWithinFilesDir(srcBaseDir, options.FilesDir); err != nil { + r.AddOnError(yamlPath, err) + continue + } + info, err := os.Stat(srcBaseDir) + if err != nil { + r.AddOnError(yamlPath, err) + continue + } + if !info.IsDir() { + r.AddOnError(yamlPath, common.ErrTreeNotDirectory) + continue + } + destBaseDir := "/" + if util.NotEmpty(tree.Path) { + destBaseDir = *tree.Path + } + + walkTree(yamlPath, &ts, &r, t, srcBaseDir, destBaseDir, options) + } + return ts, r +} + +func walkTree(yamlPath path.ContextPath, ts *translate.TranslationSet, r *report.Report, t *nodeTracker, srcBaseDir, destBaseDir string, options common.TranslateOptions) { + // The strategy for errors within WalkFunc is to add an error to + // the report and return nil, so walking continues but translation + // will fail afterward. + err := filepath.Walk(srcBaseDir, func(srcPath string, info os.FileInfo, err error) error { + if err != nil { + r.AddOnError(yamlPath, err) + return nil + } + relPath, err := filepath.Rel(srcBaseDir, srcPath) + if err != nil { + r.AddOnError(yamlPath, err) + return nil + } + destPath := filepath.Join(destBaseDir, relPath) + + if info.Mode().IsDir() { + return nil + } else if info.Mode().IsRegular() { + i, file := t.GetFile(destPath) + if file != nil { + if util.NotEmpty(file.Contents.Source) { + r.AddOnError(yamlPath, common.ErrNodeExists) + return nil + } + } else { + if t.Exists(destPath) { + r.AddOnError(yamlPath, common.ErrNodeExists) + return nil + } + i, file = t.AddFile(types.File{ + Node: types.Node{ + Path: destPath, + }, + }) + ts.AddFromCommonSource(yamlPath, path.New("json", "storage", "files", i), file) + if i == 0 { + ts.AddTranslation(yamlPath, path.New("json", "storage", "files")) + } + } + contents, err := ioutil.ReadFile(srcPath) + if err != nil { + r.AddOnError(yamlPath, err) + return nil + } + url, gzipped, err := baseutil.MakeDataURL(contents, file.Contents.Compression, !options.NoResourceAutoCompression) + if err != nil { + r.AddOnError(yamlPath, err) + return nil + } + file.Contents.Source = util.StrToPtr(url) + ts.AddTranslation(yamlPath, path.New("json", "storage", "files", i, "contents", "source")) + if gzipped { + file.Contents.Compression = util.StrToPtr("gzip") + ts.AddTranslation(yamlPath, path.New("json", "storage", "files", i, "contents", "compression")) + } + ts.AddTranslation(yamlPath, path.New("json", "storage", "files", i, "contents")) + if file.Mode == nil { + mode := 0644 + if info.Mode()&0111 != 0 { + mode = 0755 + } + file.Mode = &mode + ts.AddTranslation(yamlPath, path.New("json", "storage", "files", i, "mode")) + } + } else if info.Mode()&os.ModeType == os.ModeSymlink { + i, link := t.GetLink(destPath) + if link != nil { + if link.Target != "" { + r.AddOnError(yamlPath, common.ErrNodeExists) + return nil + } + } else { + if t.Exists(destPath) { + r.AddOnError(yamlPath, common.ErrNodeExists) + return nil + } + i, link = t.AddLink(types.Link{ + Node: types.Node{ + Path: destPath, + }, + }) + ts.AddFromCommonSource(yamlPath, path.New("json", "storage", "links", i), link) + if i == 0 { + ts.AddTranslation(yamlPath, path.New("json", "storage", "links")) + } + } + link.Target, err = os.Readlink(srcPath) + if err != nil { + r.AddOnError(yamlPath, err) + return nil + } + ts.AddTranslation(yamlPath, path.New("json", "storage", "links", i, "target")) + } else { + r.AddOnError(yamlPath, common.ErrFileType) + return nil + } + return nil + }) + r.AddOnError(yamlPath, err) +} + +func (c Config) addMountUnits(config *types.Config, ts *translate.TranslationSet) { + if len(c.Storage.Filesystems) == 0 { + return + } + var rendered types.Config + renderedTranslations := translate.NewTranslationSet("yaml", "json") + renderedTranslations.AddTranslation(path.New("yaml", "storage", "filesystems"), path.New("json", "systemd")) + renderedTranslations.AddTranslation(path.New("yaml", "storage", "filesystems"), path.New("json", "systemd", "units")) + for i, fs := range c.Storage.Filesystems { + if !util.IsTrue(fs.WithMountUnit) { + continue + } + fromPath := path.New("yaml", "storage", "filesystems", i, "with_mount_unit") + remote := false + // check filesystems targeting /dev/mapper devices against LUKS to determine if a + // remote mount is needed + if strings.HasPrefix(fs.Device, "/dev/mapper/") || strings.HasPrefix(fs.Device, "/dev/disk/by-id/dm-name-") { + for _, luks := range c.Storage.Luks { + // LUKS devices are opened with their name specified + if fs.Device == fmt.Sprintf("/dev/mapper/%s", luks.Name) || fs.Device == fmt.Sprintf("/dev/disk/by-id/dm-name-%s", luks.Name) { + if luks.Clevis != nil && len(luks.Clevis.Tang) > 0 { + remote = true + break + } + } + } + } + newUnit := mountUnitFromFS(fs, remote) + unitPath := path.New("json", "systemd", "units", len(rendered.Systemd.Units)) + rendered.Systemd.Units = append(rendered.Systemd.Units, newUnit) + renderedTranslations.AddFromCommonSource(fromPath, unitPath, newUnit) + } + retConfig, retTranslations := baseutil.MergeTranslatedConfigs(rendered, renderedTranslations, *config, *ts) + *config = retConfig.(types.Config) + *ts = retTranslations +} + +func mountUnitFromFS(fs Filesystem, remote bool) types.Unit { + context := struct { + *Filesystem + EscapedDevice string + Remote bool + }{ + Filesystem: &fs, + EscapedDevice: unit.UnitNamePathEscape(fs.Device), + Remote: remote, + } + contents := strings.Builder{} + err := mountUnitTemplate.Execute(&contents, context) + if err != nil { + panic(err) + } + // unchecked deref of path ok, fs would fail validation otherwise + unitName := unit.UnitNamePathEscape(*fs.Path) + ".mount" + return types.Unit{ + Name: unitName, + Enabled: util.BoolToPtr(true), + Contents: util.StrToPtr(contents.String()), + } +} diff --git a/vendor/github.com/coreos/butane/base/v0_3/util.go b/vendor/github.com/coreos/butane/base/v0_3/util.go new file mode 100644 index 0000000000..5d11df31d5 --- /dev/null +++ b/vendor/github.com/coreos/butane/base/v0_3/util.go @@ -0,0 +1,125 @@ +// Copyright 2020 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_3 + +import ( + "github.com/coreos/ignition/v2/config/v3_2/types" +) + +type nodeTracker struct { + files *[]types.File + fileMap map[string]int + + dirs *[]types.Directory + dirMap map[string]int + + links *[]types.Link + linkMap map[string]int +} + +func newNodeTracker(c *types.Config) *nodeTracker { + t := nodeTracker{ + files: &c.Storage.Files, + fileMap: make(map[string]int, len(c.Storage.Files)), + + dirs: &c.Storage.Directories, + dirMap: make(map[string]int, len(c.Storage.Directories)), + + links: &c.Storage.Links, + linkMap: make(map[string]int, len(c.Storage.Links)), + } + for i, n := range *t.files { + t.fileMap[n.Path] = i + } + for i, n := range *t.dirs { + t.dirMap[n.Path] = i + } + for i, n := range *t.links { + t.linkMap[n.Path] = i + } + return &t +} + +func (t *nodeTracker) Exists(path string) bool { + for _, m := range []map[string]int{t.fileMap, t.dirMap, t.linkMap} { + if _, ok := m[path]; ok { + return true + } + } + return false +} + +func (t *nodeTracker) GetFile(path string) (int, *types.File) { + if i, ok := t.fileMap[path]; ok { + return i, &(*t.files)[i] + } else { + return 0, nil + } +} + +func (t *nodeTracker) AddFile(f types.File) (int, *types.File) { + if f.Path == "" { + panic("File path missing") + } + if _, ok := t.fileMap[f.Path]; ok { + panic("Adding already existing file") + } + i := len(*t.files) + *t.files = append(*t.files, f) + t.fileMap[f.Path] = i + return i, &(*t.files)[i] +} + +func (t *nodeTracker) GetDir(path string) (int, *types.Directory) { + if i, ok := t.dirMap[path]; ok { + return i, &(*t.dirs)[i] + } else { + return 0, nil + } +} + +func (t *nodeTracker) AddDir(d types.Directory) (int, *types.Directory) { + if d.Path == "" { + panic("Directory path missing") + } + if _, ok := t.dirMap[d.Path]; ok { + panic("Adding already existing directory") + } + i := len(*t.dirs) + *t.dirs = append(*t.dirs, d) + t.dirMap[d.Path] = i + return i, &(*t.dirs)[i] +} + +func (t *nodeTracker) GetLink(path string) (int, *types.Link) { + if i, ok := t.linkMap[path]; ok { + return i, &(*t.links)[i] + } else { + return 0, nil + } +} + +func (t *nodeTracker) AddLink(l types.Link) (int, *types.Link) { + if l.Path == "" { + panic("Link path missing") + } + if _, ok := t.linkMap[l.Path]; ok { + panic("Adding already existing link") + } + i := len(*t.links) + *t.links = append(*t.links, l) + t.linkMap[l.Path] = i + return i, &(*t.links)[i] +} diff --git a/vendor/github.com/coreos/butane/base/v0_3/validate.go b/vendor/github.com/coreos/butane/base/v0_3/validate.go new file mode 100644 index 0000000000..9a00c354bf --- /dev/null +++ b/vendor/github.com/coreos/butane/base/v0_3/validate.go @@ -0,0 +1,79 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v0_3 + +import ( + baseutil "github.com/coreos/butane/base/util" + "github.com/coreos/butane/config/common" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +func (rs Resource) Validate(c path.ContextPath) (r report.Report) { + var field string + sources := 0 + if rs.Local != nil { + sources++ + field = "local" + } + if rs.Inline != nil { + sources++ + field = "inline" + } + if rs.Source != nil { + sources++ + field = "source" + } + if sources > 1 { + r.AddOnError(c.Append(field), common.ErrTooManyResourceSources) + } + return +} + +func (fs Filesystem) Validate(c path.ContextPath) (r report.Report) { + if !util.IsTrue(fs.WithMountUnit) { + return + } + if util.NilOrEmpty(fs.Path) { + r.AddOnError(c.Append("path"), common.ErrMountUnitNoPath) + } + if util.NilOrEmpty(fs.Format) { + r.AddOnError(c.Append("format"), common.ErrMountUnitNoFormat) + } + return +} + +func (d Directory) Validate(c path.ContextPath) (r report.Report) { + if d.Mode != nil { + r.AddOnWarn(c.Append("mode"), baseutil.CheckForDecimalMode(*d.Mode, true)) + } + return +} + +func (f File) Validate(c path.ContextPath) (r report.Report) { + if f.Mode != nil { + r.AddOnWarn(c.Append("mode"), baseutil.CheckForDecimalMode(*f.Mode, false)) + } + return +} + +func (t Tree) Validate(c path.ContextPath) (r report.Report) { + if t.Local == "" { + r.AddOnError(c, common.ErrTreeNoLocal) + } + return +} diff --git a/vendor/github.com/coreos/butane/config/common/common.go b/vendor/github.com/coreos/butane/config/common/common.go new file mode 100644 index 0000000000..d23cdc6010 --- /dev/null +++ b/vendor/github.com/coreos/butane/config/common/common.go @@ -0,0 +1,27 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package common + +type TranslateOptions struct { + FilesDir string // allow embedding local files relative to this directory + NoResourceAutoCompression bool // skip automatic compression of inline/local resources + DebugPrintTranslations bool // report translations to stderr +} + +type TranslateBytesOptions struct { + TranslateOptions + Pretty bool + Raw bool // encode only the Ignition config, not any wrapper +} diff --git a/vendor/github.com/coreos/butane/config/common/errors.go b/vendor/github.com/coreos/butane/config/common/errors.go new file mode 100644 index 0000000000..38313835b1 --- /dev/null +++ b/vendor/github.com/coreos/butane/config/common/errors.go @@ -0,0 +1,75 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package common + +import ( + "errors" +) + +var ( + // common field parsing + ErrNoVariant = errors.New("error parsing variant; must be specified") + ErrInvalidVersion = errors.New("error parsing version; must be a valid semver") + + // high-level errors for fatal reports + ErrInvalidSourceConfig = errors.New("source config is invalid") + ErrInvalidGeneratedConfig = errors.New("config generated was invalid") + + // deprecated variant/version + ErrRhcosVariantDeprecated = errors.New("this variant is deprecated and will be removed in a future release; use openshift variant instead") + + // resources and trees + ErrTooManyResourceSources = errors.New("only one of the following can be set: inline, local, source") + ErrFilesDirEscape = errors.New("local file path traverses outside the files directory") + ErrFileType = errors.New("trees may only contain files, directories, and symlinks") + ErrNodeExists = errors.New("matching filesystem node has existing contents or different type") + ErrNoFilesDir = errors.New("local file paths are relative to a files directory that must be specified with -d/--files-dir") + ErrTreeNotDirectory = errors.New("root of tree must be a directory") + ErrTreeNoLocal = errors.New("local is required") + + // filesystem nodes + ErrDecimalMode = errors.New("unreasonable mode would be reasonable if specified in octal; remember to add a leading zero") + + // mount units + ErrMountUnitNoPath = errors.New("path is required if with_mount_unit is true and format is not swap") + ErrMountUnitNoFormat = errors.New("format is required if with_mount_unit is true") + + // boot device + ErrUnknownBootDeviceLayout = errors.New("layout must be one of: aarch64, ppc64le, x86_64") + ErrTooFewMirrorDevices = errors.New("mirroring requires at least two devices") + + // partition + ErrWrongPartitionNumber = errors.New("incorrect partition number; a new partition will be created using reserved label") + + // MachineConfigs + ErrFieldElided = errors.New("field ignored in raw mode") + ErrNameRequired = errors.New("metadata.name is required") + ErrRoleRequired = errors.New("machineconfiguration.openshift.io/role label is required") + ErrInvalidKernelType = errors.New("must be empty, \"default\", or \"realtime\"") + ErrBtrfsSupport = errors.New("btrfs is not supported in this spec version") + ErrFilesystemNoneSupport = errors.New("format \"none\" is not supported in this spec version") + ErrDirectorySupport = errors.New("directories are not supported in this spec version") + ErrFileSchemeSupport = errors.New("file contents source must be data URL in this spec version") + ErrFileAppendSupport = errors.New("appending to files is not supported in this spec version") + ErrFileCompressionSupport = errors.New("file compression is not supported in this spec version") + ErrLinkSupport = errors.New("links are not supported in this spec version") + ErrGroupSupport = errors.New("groups are not supported in this spec version") + ErrUserFieldSupport = errors.New("fields other than \"name\" and \"ssh_authorized_keys\" are not supported in this spec version") + ErrUserNameSupport = errors.New("users other than \"core\" are not supported in this spec version") + ErrKernelArgumentSupport = errors.New("this field cannot be used for kernel arguments in this spec version; use openshift.kernel_arguments instead") + + // Extensions + ErrExtensionNameRequired = errors.New("field \"name\" is required") +) diff --git a/vendor/github.com/coreos/fcct/base/v0_1/validate.go b/vendor/github.com/coreos/butane/config/fcos/v1_2/schema.go similarity index 58% rename from vendor/github.com/coreos/fcct/base/v0_1/validate.go rename to vendor/github.com/coreos/butane/config/fcos/v1_2/schema.go index c8e6942aa4..42fb946823 100644 --- a/vendor/github.com/coreos/fcct/base/v0_1/validate.go +++ b/vendor/github.com/coreos/butane/config/fcos/v1_2/schema.go @@ -1,4 +1,4 @@ -// Copyright 2019 Red Hat, Inc +// Copyright 2020 Red Hat, Inc // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,22 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License.) -package v0_1 +package v1_2 import ( - "errors" - - "github.com/coreos/vcontext/path" - "github.com/coreos/vcontext/report" -) - -var ( - ErrInlineAndSource = errors.New("inline cannot be specified if source is specified") + base "github.com/coreos/butane/base/v0_3" ) -func (f FileContents) Validate(c path.ContextPath) (r report.Report) { - if f.Inline != nil && f.Source != nil { - r.AddOnError(c.Append("inline"), ErrInlineAndSource) - } - return +type Config struct { + base.Config `yaml:",inline"` } diff --git a/vendor/github.com/coreos/butane/config/fcos/v1_2/translate.go b/vendor/github.com/coreos/butane/config/fcos/v1_2/translate.go new file mode 100644 index 0000000000..cff6eb1dc1 --- /dev/null +++ b/vendor/github.com/coreos/butane/config/fcos/v1_2/translate.go @@ -0,0 +1,68 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package v1_2 + +import ( + "github.com/coreos/butane/config/common" + cutil "github.com/coreos/butane/config/util" + "github.com/coreos/butane/translate" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/ignition/v2/config/v3_2/types" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +// ToIgn3_2Unvalidated translates the config to an Ignition config. It also +// returns the set of translations it did so paths in the resultant config +// can be tracked back to their source in the source config. No config +// validation is performed on input or output. +func (c Config) ToIgn3_2Unvalidated(options common.TranslateOptions) (types.Config, translate.TranslationSet, report.Report) { + ret, ts, r := c.Config.ToIgn3_2Unvalidated(options) + if r.IsFatal() { + return types.Config{}, translate.TranslationSet{}, r + } + + for i, disk := range ret.Storage.Disks { + // Don't warn if wipeTable is set, matching later spec versions + if !util.IsTrue(disk.WipeTable) { + for j, partition := range disk.Partitions { + // check for reserved partlabels + if partition.Label != nil { + if (*partition.Label == "BIOS-BOOT" && partition.Number != 1) || (*partition.Label == "PowerPC-PReP-boot" && partition.Number != 1) || (*partition.Label == "EFI-SYSTEM" && partition.Number != 2) || (*partition.Label == "boot" && partition.Number != 3) || (*partition.Label == "root" && partition.Number != 4) { + r.AddOnWarn(path.New("json", "storage", "disks", i, "partitions", j, "label"), common.ErrWrongPartitionNumber) + } + } + } + } + } + return ret, ts, r +} + +// ToIgn3_2 translates the config to an Ignition config. It returns a +// report of any errors or warnings in the source and resultant config. If +// the report has fatal errors or it encounters other problems translating, +// an error is returned. +func (c Config) ToIgn3_2(options common.TranslateOptions) (types.Config, report.Report, error) { + cfg, r, err := cutil.Translate(c, "ToIgn3_2Unvalidated", options) + return cfg.(types.Config), r, err +} + +// ToIgn3_2Bytes translates from a v1.2 Butane config to a v3.2.0 Ignition config. It returns a report of any errors or +// warnings in the source and resultant config. If the report has fatal errors or it encounters other problems +// translating, an error is returned. +func ToIgn3_2Bytes(input []byte, options common.TranslateBytesOptions) ([]byte, report.Report, error) { + return cutil.TranslateBytes(input, &Config{}, "ToIgn3_2", options) +} diff --git a/vendor/github.com/coreos/butane/config/util/util.go b/vendor/github.com/coreos/butane/config/util/util.go new file mode 100644 index 0000000000..64599c4099 --- /dev/null +++ b/vendor/github.com/coreos/butane/config/util/util.go @@ -0,0 +1,246 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package util + +import ( + "bytes" + "fmt" + "os" + "reflect" + "regexp" + "strings" + + "github.com/coreos/butane/config/common" + "github.com/coreos/butane/translate" + + "github.com/clarketm/json" + ignvalidate "github.com/coreos/ignition/v2/config/validate" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" + "github.com/coreos/vcontext/tree" + "github.com/coreos/vcontext/validate" + vyaml "github.com/coreos/vcontext/yaml" + "gopkg.in/yaml.v3" +) + +var ( + snakeRe = regexp.MustCompile("([A-Z])") +) + +// Misc helpers + +// Translate translates cfg to the corresponding Ignition config version +// using the named translation method on cfg, and returns the marshaled +// Ignition config. It returns a report of any errors or warnings in the +// source and resultant config. If the report has fatal errors or it +// encounters other problems translating, an error is returned. +func Translate(cfg interface{}, translateMethod string, options common.TranslateOptions) (interface{}, report.Report, error) { + // Get method, and zero return value for error returns. + method := reflect.ValueOf(cfg).MethodByName(translateMethod) + zeroValue := reflect.Zero(method.Type().Out(0)).Interface() + + // Validate the input. + r := validate.Validate(cfg, "yaml") + if r.IsFatal() { + return zeroValue, r, common.ErrInvalidSourceConfig + } + + // Perform the translation. + translateRet := method.Call([]reflect.Value{reflect.ValueOf(options)}) + final := translateRet[0].Interface() + translations := translateRet[1].Interface().(translate.TranslationSet) + translateReport := translateRet[2].Interface().(report.Report) + r.Merge(translateReport) + if r.IsFatal() { + return zeroValue, r, common.ErrInvalidSourceConfig + } + if options.DebugPrintTranslations { + fmt.Fprint(os.Stderr, translations) + if err := translations.DebugVerifyCoverage(final); err != nil { + fmt.Fprintf(os.Stderr, "\n%s", err) + } + } + + // Check for invalid duplicated keys. + dupsReport := validate.ValidateCustom(final, "json", ignvalidate.ValidateDups) + r.Merge(TranslateReportPaths(dupsReport, translations)) + + // Validate JSON semantics. + jsonReport := validate.Validate(final, "json") + r.Merge(TranslateReportPaths(jsonReport, translations)) + + if r.IsFatal() { + return zeroValue, r, common.ErrInvalidGeneratedConfig + } + return final, r, nil +} + +// TranslateBytes unmarshals the Butane config specified in input into the +// struct pointed to by container, translates it to the corresponding Ignition +// config version using the named translation method, and returns the +// marshaled Ignition config. It returns a report of any errors or warnings +// in the source and resultant config. If the report has fatal errors or it +// encounters other problems translating, an error is returned. +func TranslateBytes(input []byte, container interface{}, translateMethod string, options common.TranslateBytesOptions) ([]byte, report.Report, error) { + cfg := container + + // Unmarshal the YAML. + contextTree, err := unmarshal(input, cfg) + if err != nil { + return nil, report.Report{}, err + } + + // Check for unused keys. + unusedKeyCheck := func(v reflect.Value, c path.ContextPath) report.Report { + return ignvalidate.ValidateUnusedKeys(v, c, contextTree) + } + r := validate.ValidateCustom(cfg, "yaml", unusedKeyCheck) + r.Correlate(contextTree) + if r.IsFatal() { + return nil, r, common.ErrInvalidSourceConfig + } + + // Perform the translation. + translateRet := reflect.ValueOf(cfg).MethodByName(translateMethod).Call([]reflect.Value{reflect.ValueOf(options.TranslateOptions)}) + final := translateRet[0].Interface() + translateReport := translateRet[1].Interface().(report.Report) + errVal := translateRet[2] + translateReport.Correlate(contextTree) + r.Merge(translateReport) + if !errVal.IsNil() { + return nil, r, errVal.Interface().(error) + } + if r.IsFatal() { + return nil, r, common.ErrInvalidSourceConfig + } + + // Marshal the JSON. + outbytes, err := marshal(final, options.Pretty) + return outbytes, r, err +} + +func TranslateBytesYAML(input []byte, container interface{}, translateMethod string, options common.TranslateBytesOptions) ([]byte, report.Report, error) { + // marshal to JSON, unmarshal, remarshal to YAML. there's no other + // good way to respect the `json` struct tags. + // https://github.com/go-yaml/yaml/issues/424 + jsonCfg, r, err := TranslateBytes(input, container, translateMethod, options) + if err != nil { + return jsonCfg, r, err + } + + var ifaceCfg interface{} + if err := json.Unmarshal(jsonCfg, &ifaceCfg); err != nil { + return []byte{}, r, err + } + + var yamlCfgBuf bytes.Buffer + yamlCfgBuf.WriteString("# Generated by Butane; do not edit\n") + encoder := yaml.NewEncoder(&yamlCfgBuf) + encoder.SetIndent(2) + if err := encoder.Encode(ifaceCfg); err != nil { + return []byte{}, r, err + } + if err := encoder.Close(); err != nil { + return []byte{}, r, err + } + yamlCfg := bytes.Trim(yamlCfgBuf.Bytes(), "\n") + return yamlCfg, r, err +} + +// Report an ErrFieldElided warning for any non-zero top-level fields in the +// specified output struct. The caller will probably want to use +// translate.PrefixReport() to reparent the report into the right place in +// the `json` hierarchy, and then TranslateReportPaths() to map back into +// `yaml` space. +func CheckForElidedFields(struct_ interface{}) report.Report { + v := reflect.ValueOf(struct_) + t := v.Type() + if t.Kind() != reflect.Struct { + panic("struct type required") + } + var r report.Report + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + if f.IsValid() && !f.IsZero() { + tag := strings.Split(t.Field(i).Tag.Get("json"), ",")[0] + r.AddOnWarn(path.New("json", tag), common.ErrFieldElided) + } + } + return r +} + +// unmarshal unmarshals the data to "to" and also returns a context tree for the source. +func unmarshal(data []byte, to interface{}) (tree.Node, error) { + dec := yaml.NewDecoder(bytes.NewReader(data)) + if err := dec.Decode(to); err != nil { + return nil, err + } + return vyaml.UnmarshalToContext(data) +} + +// marshal is a wrapper for marshaling to json with or without pretty-printing the output +func marshal(from interface{}, pretty bool) ([]byte, error) { + if pretty { + return json.MarshalIndent(from, "", " ") + } + return json.Marshal(from) +} + +// snakePath converts a path.ContextPath with camelCase elements and returns the +// same path but with snake_case elements instead +func snakePath(p path.ContextPath) path.ContextPath { + ret := path.New(p.Tag) + for _, part := range p.Path { + if str, ok := part.(string); ok { + ret = ret.Append(snake(str)) + } else { + ret = ret.Append(part) + } + } + return ret +} + +// snake converts from camelCase (not CamelCase) to snake_case +func snake(in string) string { + return strings.ToLower(snakeRe.ReplaceAllString(in, "_$1")) +} + +// TranslateReportPaths takes a report from a camelCase json document and a set of translations rules, +// applies those rules and converts all camelCase to snake_case. +func TranslateReportPaths(r report.Report, ts translate.TranslationSet) report.Report { + var ret report.Report + ret.Merge(r) + for i, ent := range ret.Entries { + context := ent.Context + if t, ok := ts.Set[context.String()]; ok { + context = t.From + } else { + // Missing translation. As a fallback, convert + // camelCase path elements to snake_case and hope + // there's a 1:1 mapping between the YAML and JSON + // hierarchies. Notably, that's not true for + // MachineConfig output, since the Ignition config + // is reparented to a grandchild of the root. + // See also https://github.com/coreos/butane/issues/213. + // This is hacky (notably, it leaves context.Tag as + // `json`) but sometimes it's enough to help us find + // a Marker, and when it isn't, the path still + // provides some feedback to the user. + context = snakePath(context) + } + ret.Entries[i].Context = context + } + return ret +} diff --git a/vendor/github.com/coreos/fcct/translate/set.go b/vendor/github.com/coreos/butane/translate/set.go similarity index 52% rename from vendor/github.com/coreos/fcct/translate/set.go rename to vendor/github.com/coreos/butane/translate/set.go index 6140fbf1b4..e6c582e4c9 100644 --- a/vendor/github.com/coreos/fcct/translate/set.go +++ b/vendor/github.com/coreos/butane/translate/set.go @@ -17,6 +17,8 @@ package translate import ( "fmt" "reflect" + "sort" + "strings" "github.com/coreos/vcontext/path" ) @@ -29,6 +31,10 @@ type Translation struct { To path.ContextPath } +func (t Translation) String() string { + return fmt.Sprintf("%s → %s", t.From, t.To) +} + // TranslationSet represents all of the translations that occurred. They're stored in a map from a string representation // of the destination path to the translation struct. The map is purely an optimization to allow fast lookups. Ideally the // map would just be from the destination path.ContextPath to the source path.ContextPath, but ContextPath contains a slice @@ -48,9 +54,28 @@ func NewTranslationSet(fromTag, toTag string) TranslationSet { } func (ts TranslationSet) String() string { - str := fmt.Sprintf("from: %v\nto: %v\n", ts.FromTag, ts.ToTag) + type entry struct { + sortKey string + formatted string + } + var entries []entry for k, v := range ts.Set { - str += fmt.Sprintf("%s: %v -> %v\n", k, v.From.String(), v.To.String()) + formatted := v.String() + // lookup key should always match To path; report if it doesn't + if k != v.To.String() { + formatted += fmt.Sprintf(" (key: %s)", k) + } + entries = append(entries, entry{ + sortKey: v.To.String(), + formatted: formatted, + }) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].sortKey < entries[j].sortKey + }) + str := fmt.Sprintf("TranslationSet: %v → %v\n", ts.FromTag, ts.ToTag) + for _, entry := range entries { + str += entry.formatted + "\n" } return str } @@ -68,24 +93,16 @@ func (ts TranslationSet) AddTranslation(from, to path.ContextPath) { ts.Set[toString] = translation } -// Shortcut for AddTranslation for identity translations -func (ts TranslationSet) AddIdentity(paths ...string) { - for _, p := range paths { - from := path.New(ts.FromTag, p) - to := path.New(ts.ToTag, p) - ts.AddTranslation(from, to) - } -} - // AddFromCommonSource adds translations for all of the paths in to from a single common path. This is useful // if one part of a config generates a large struct and all of the large struct should map to one path in the // config being translated. func (ts TranslationSet) AddFromCommonSource(common path.ContextPath, toPrefix path.ContextPath, to interface{}) { v := reflect.ValueOf(to) - vPaths := prefixPaths(getAllPaths(v, ts.ToTag), toPrefix.Path...) - for _, path := range vPaths { - ts.AddTranslation(common, path) + vPaths := prefixPaths(getAllPaths(v, ts.ToTag, true), toPrefix.Path...) + for _, toPath := range vPaths { + ts.AddTranslation(common, toPath) } + ts.AddTranslation(common, toPrefix) } // Merge adds all the entries to the set. It mutates the Set in place. @@ -95,19 +112,66 @@ func (ts TranslationSet) Merge(from TranslationSet) { } } -// MergeP is like Merge, but first it calls Prefix on the set being merged in. +// MergeP is like Merge, but it adds a prefix to the set being merged in. func (ts TranslationSet) MergeP(prefix interface{}, from TranslationSet) { - from = from.Prefix(prefix) + ts.MergeP2(prefix, prefix, from) +} + +// MergeP2 is like Merge, but it adds distinct prefixes to each side of the +// set being merged in. +func (ts TranslationSet) MergeP2(fromPrefix interface{}, toPrefix interface{}, from TranslationSet) { + from = from.PrefixPaths(path.New(from.FromTag, fromPrefix), path.New(from.ToTag, toPrefix)) ts.Merge(from) } // Prefix returns a TranslationSet with all translation paths prefixed by prefix. func (ts TranslationSet) Prefix(prefix interface{}) TranslationSet { + return ts.PrefixPaths(path.New(ts.FromTag, prefix), path.New(ts.ToTag, prefix)) +} + +// PrefixPaths returns a TranslationSet with from translation paths prefixed by +// fromPrefix and to translation paths prefixed by toPrefix. +func (ts TranslationSet) PrefixPaths(fromPrefix, toPrefix path.ContextPath) TranslationSet { ret := NewTranslationSet(ts.FromTag, ts.ToTag) - from := path.New(ts.FromTag, prefix) - to := path.New(ts.ToTag, prefix) for _, tr := range ts.Set { - ret.AddTranslation(from.Append(tr.From.Path...), to.Append(tr.From.Path...)) + ret.AddTranslation(fromPrefix.Append(tr.From.Path...), toPrefix.Append(tr.To.Path...)) } return ret } + +// Descend returns the subtree of translations rooted at the specified To path. +func (ts TranslationSet) Descend(to path.ContextPath) TranslationSet { + ret := NewTranslationSet(ts.FromTag, ts.ToTag) +OUTER: + for _, tr := range ts.Set { + if len(tr.To.Path) < len(to.Path) { + // can't be in the requested subtree; skip + continue + } + for i, e := range to.Path { + if tr.To.Path[i] != e { + // not in the requested subtree; skip + continue OUTER + } + } + subtreePath := path.New(tr.To.Tag, tr.To.Path[len(to.Path):]...) + ret.AddTranslation(tr.From, subtreePath) + } + return ret +} + +// DebugVerifyCoverage recursively checks whether every non-zero field in v +// has a translation. If translations are missing, it returns a multi-line +// error listing them. +func (ts TranslationSet) DebugVerifyCoverage(v interface{}) error { + var missingPaths []string + for _, pathToCheck := range getAllPaths(reflect.ValueOf(v), ts.ToTag, false) { + if _, ok := ts.Set[pathToCheck.String()]; !ok { + missingPaths = append(missingPaths, pathToCheck.String()) + } + } + if len(missingPaths) > 0 { + return fmt.Errorf("Missing paths in TranslationSet:\n%v\n", strings.Join(missingPaths, "\n")) + } + return nil +} diff --git a/vendor/github.com/coreos/fcct/translate/translate.go b/vendor/github.com/coreos/butane/translate/translate.go similarity index 76% rename from vendor/github.com/coreos/fcct/translate/translate.go rename to vendor/github.com/coreos/butane/translate/translate.go index 3cbfbf4594..8977b4740b 100644 --- a/vendor/github.com/coreos/fcct/translate/translate.go +++ b/vendor/github.com/coreos/butane/translate/translate.go @@ -20,6 +20,7 @@ import ( "github.com/coreos/ignition/v2/config/util" "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" ) /* @@ -28,14 +29,20 @@ import ( * call NewTranslator() to get a translator instance. This can then have * additional translation rules (in the form of functions) to translate from * types in one struct to the other. Those functions are in the form: - * func(typeFromInputStruct) -> typeFromOutputStruct + * func(fromType, optionsType) -> (toType, TranslationSet, report.Report) * These can be closures that reference the translator as well. This allows for * manually translating some fields but resuming automatic translation on the * other fields through the Translator.Translate() function. */ +const ( + TAG_KEY = "butane" + TAG_AUTO_SKIP = "auto_skip" +) + var ( translationsType = reflect.TypeOf(TranslationSet{}) + reportType = reflect.TypeOf(report.Report{}) ) // Returns if this type can be translated without a custom translator. Children or other @@ -62,13 +69,19 @@ func (t translator) translatable(t1, t2 reflect.Type) bool { // precondition: t1, t2 are both of Kind 'struct' func (t translator) translatableStruct(t1, t2 reflect.Type) bool { - if t1.NumField() != t2.NumField() || t1.Name() != t2.Name() { + if t1.Name() != t2.Name() { return false } + t1Fields := 0 for i := 0; i < t1.NumField(); i++ { t1f := t1.Field(i) - t2f, ok := t2.FieldByName(t1f.Name) + if t1f.Tag.Get(TAG_KEY) == TAG_AUTO_SKIP { + // ignore this input field + continue + } + t1Fields++ + t2f, ok := t2.FieldByName(t1f.Name) if !ok { return false } @@ -76,20 +89,22 @@ func (t translator) translatableStruct(t1, t2 reflect.Type) bool { return false } } - return true + return t2.NumField() == t1Fields } // checks that t could reasonably be the type of a translator function -func couldBeValidTranslator(t reflect.Type) bool { - if t.Kind() != reflect.Func { +func (t translator) couldBeValidTranslator(tr reflect.Type) bool { + if tr.Kind() != reflect.Func { return false } - if t.NumIn() != 1 || t.NumOut() != 2 { + if tr.NumIn() != 2 || tr.NumOut() != 3 { return false } - if util.IsInvalidInConfig(t.In(0).Kind()) || - util.IsInvalidInConfig(t.Out(0).Kind()) || - t.Out(1) != translationsType { + if util.IsInvalidInConfig(tr.In(0).Kind()) || + util.IsInvalidInConfig(tr.Out(0).Kind()) || + tr.In(1) != reflect.TypeOf(t.options) || + tr.Out(1) != translationsType || + tr.Out(2) != reportType { return false } return true @@ -120,8 +135,13 @@ func (t translator) translateSameType(vFrom, vTo reflect.Value, fromPath, toPath for i := 0; i < vFrom.Len(); i++ { t.translate(vFrom.Index(i), vTo.Index(i), fromPath.Append(i), toPath.Append(i)) } + t.translations.AddTranslation(fromPath, toPath) case k == reflect.Struct: for i := 0; i < vFrom.NumField(); i++ { + if vFrom.Type().Field(i).Tag.Get(TAG_KEY) == TAG_AUTO_SKIP { + // ignore this input field + continue + } fieldGoName := vFrom.Type().Field(i).Name toStructField, ok := vTo.Type().FieldByName(fieldGoName) if !ok { @@ -138,6 +158,9 @@ func (t translator) translateSameType(vFrom, vTo reflect.Value, fromPath, toPath } t.translate(vFrom.Field(i), vToField, from, to) } + if !vFrom.IsZero() { + t.translations.AddTranslation(fromPath, toPath) + } default: panic("Encountered types that are not the same when they should be. This is a bug, please file a report") } @@ -153,16 +176,23 @@ func (t translator) translate(vFrom, vTo reflect.Value, fromPath, toPath path.Co tFrom := vFrom.Type() tTo := vTo.Type() if fnv := t.getTranslator(tFrom, tTo); fnv.IsValid() { - returns := fnv.Call([]reflect.Value{vFrom}) + returns := fnv.Call([]reflect.Value{vFrom, reflect.ValueOf(t.options)}) vTo.Set(returns[0]) // handle all the translations and "rebase" them to our current place retSet := returns[1].Interface().(TranslationSet) - for _, trans := range retSet.Set { - from := fromPath.Append(trans.From.Path...) - to := toPath.Append(trans.To.Path...) - t.translations.AddTranslation(from, to) + t.translations.Merge(retSet.PrefixPaths(fromPath, toPath)) + if len(retSet.Set) > 0 { + t.translations.AddTranslation(fromPath, toPath) + } + + // likewise for the report entries + retReport := returns[2].Interface().(report.Report) + for i := range retReport.Entries { + entry := &retReport.Entries[i] + entry.Context = fromPath.Append(entry.Context.Path...) } + t.report.Merge(retReport) return } if t.translatable(tFrom, tTo) { @@ -175,18 +205,19 @@ func (t translator) translate(vFrom, vTo reflect.Value, fromPath, toPath path.Co type Translator interface { // Adds a custom translator for cases where the structs are not identical. Must be of type - // func(fromType) -> (toType, TranslationSet). The translator should return the set of all - // translations it did. + // func(fromType, optionsType) -> (toType, TranslationSet, report.Report). + // The translator should return the set of all translations it did. AddCustomTranslator(t interface{}) // Also returns a list of source and dest paths, autocompleted by fromTag and toTag - Translate(from, to interface{}) TranslationSet + Translate(from, to interface{}) (TranslationSet, report.Report) } // NewTranslator creates a new Translator for translating from types with fromTag struct tags (e.g. "yaml") // to types with toTag struct tages (e.g. "json"). These tags are used when determining paths when generating // the TranslationSet returned by Translator.Translate() -func NewTranslator(fromTag, toTag string) Translator { +func NewTranslator(fromTag, toTag string, options interface{}) Translator { return &translator{ + options: options, translations: TranslationSet{ FromTag: fromTag, ToTag: toTag, @@ -196,18 +227,20 @@ func NewTranslator(fromTag, toTag string) Translator { } type translator struct { + options interface{} // List of custom translation funcs, must pass couldBeValidTranslator // This is only for fields that cannot or should not be trivially translated, // All trivially translated fields use the default behavior. translators []reflect.Value translations TranslationSet + report *report.Report } -// fn should be of the form func(fromType, translationsMap) -> toType -// fn should mutate translationsMap to add all the translations it did +// fn should be of the form +// func(fromType, optionsType) -> (toType, TranslationSet, report.Report) func (t *translator) AddCustomTranslator(fn interface{}) { fnv := reflect.ValueOf(fn) - if !couldBeValidTranslator(fnv.Type()) { + if !t.couldBeValidTranslator(fnv.Type()) { panic("Tried to register invalid translator function") } t.translators = append(t.translators, fnv) @@ -223,7 +256,7 @@ func (t translator) getTranslator(from, to reflect.Type) reflect.Value { } // Translate translates from into to and returns a set of all the path changes it performed. -func (t translator) Translate(from, to interface{}) TranslationSet { +func (t translator) Translate(from, to interface{}) (TranslationSet, report.Report) { fv := reflect.ValueOf(from) tv := reflect.ValueOf(to) if fv.Kind() != reflect.Ptr || tv.Kind() != reflect.Ptr { @@ -231,12 +264,13 @@ func (t translator) Translate(from, to interface{}) TranslationSet { } fv = fv.Elem() tv = tv.Elem() - // Make sure to clear this every time` + // Make sure to clear these every time t.translations = TranslationSet{ FromTag: t.translations.FromTag, ToTag: t.translations.ToTag, Set: map[string]Translation{}, } + t.report = &report.Report{} t.translate(fv, tv, path.New(t.translations.FromTag), path.New(t.translations.ToTag)) - return t.translations + return t.translations, *t.report } diff --git a/vendor/github.com/coreos/butane/translate/util.go b/vendor/github.com/coreos/butane/translate/util.go new file mode 100644 index 0000000000..71f0b40e83 --- /dev/null +++ b/vendor/github.com/coreos/butane/translate/util.go @@ -0,0 +1,133 @@ +// Copyright 2019 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translate + +import ( + "reflect" + "strings" + + "github.com/coreos/ignition/v2/config/util" + "github.com/coreos/vcontext/path" + "github.com/coreos/vcontext/report" +) + +// fieldName returns the name uses when (un)marshalling a field. t should be a reflect.Value of a struct, +// index is the field index, and tag is the struct tag used when (un)marshalling (e.g. "json" or "yaml") +func fieldName(t reflect.Value, index int, tag string) string { + f := t.Type().Field(index) + if tag == "" { + return f.Name + } + return strings.Split(f.Tag.Get(tag), ",")[0] +} + +func prefixPath(p path.ContextPath, prefix ...interface{}) path.ContextPath { + return path.New(p.Tag, prefix...).Append(p.Path...) +} + +func prefixPaths(ps []path.ContextPath, prefix ...interface{}) []path.ContextPath { + ret := []path.ContextPath{} + for _, p := range ps { + ret = append(ret, prefixPath(p, prefix...)) + } + return ret +} + +func getAllPaths(v reflect.Value, tag string, includeZeroFields bool) []path.ContextPath { + k := v.Kind() + t := v.Type() + switch { + case util.IsPrimitive(k): + return nil + case k == reflect.Ptr: + if v.IsNil() { + return nil + } + return getAllPaths(v.Elem(), tag, includeZeroFields) + case k == reflect.Slice: + ret := []path.ContextPath{} + for i := 0; i < v.Len(); i++ { + // for struct, pointer to struct, etc., add any children + ret = append(ret, prefixPaths(getAllPaths(v.Index(i), tag, includeZeroFields), i)...) + // add slice entry + ret = append(ret, path.New(tag, i)) + } + return ret + case k == reflect.Struct: + ret := []path.ContextPath{} + for i := 0; i < t.NumField(); i++ { + name := fieldName(v, i, tag) + field := v.Field(i) + if !includeZeroFields && field.IsZero() { + continue + } + if t.Field(i).Anonymous { + ret = append(ret, getAllPaths(field, tag, includeZeroFields)...) + } else { + ret = append(ret, prefixPaths(getAllPaths(field, tag, includeZeroFields), name)...) + ret = append(ret, path.New(tag, name)) + } + } + return ret + case k == reflect.Map: + // we don't have these in Butane or Ignition configs, but + // we need to support validating translations of + // metadata.labels in MachineConfig output + ret := []path.ContextPath{} + iter := v.MapRange() + for iter.Next() { + // for struct, pointer to struct, etc., add any children + ret = append(ret, prefixPaths(getAllPaths(iter.Value(), tag, includeZeroFields), iter.Key())...) + // add map entry + ret = append(ret, path.New(tag, iter.Key())) + } + return ret + default: + panic("Encountered unexpected type. This is a bug, please file a report") + } +} + +// Return a copy of the report, with the context paths prefixed by prefix. +func PrefixReport(r report.Report, prefix interface{}) report.Report { + var ret report.Report + ret.Merge(r) + for i := range ret.Entries { + entry := &ret.Entries[i] + entry.Context = path.New(entry.Context.Tag, prefix).Append(entry.Context.Path...) + } + return ret +} + +// Utility function to run a translation and prefix the resulting +// TranslationSet and Report. +func Prefixed(tr Translator, prefix interface{}, from interface{}, to interface{}) (TranslationSet, report.Report) { + tm, r := tr.Translate(from, to) + return tm.Prefix(prefix), PrefixReport(r, prefix) +} + +// Utility function to run a translation and merge the result, with the +// specified prefix, into the specified TranslationSet and Report. +func MergeP(tr Translator, tm TranslationSet, r *report.Report, prefix interface{}, from interface{}, to interface{}) { + MergeP2(tr, tm, r, prefix, from, prefix, to) +} + +// Utility function to run a translation and merge the result, with the +// specified prefixes, into the specified TranslationSet and Report. +func MergeP2(tr Translator, tm TranslationSet, r *report.Report, fromPrefix interface{}, from interface{}, toPrefix interface{}, to interface{}) { + translations, translationReport := tr.Translate(from, to) + tm.MergeP2(fromPrefix, toPrefix, translations) + // translation report paths are on the from side + r.Merge(PrefixReport(translationReport, fromPrefix)) +} diff --git a/vendor/github.com/coreos/fcct/base/v0_1/translate.go b/vendor/github.com/coreos/fcct/base/v0_1/translate.go deleted file mode 100644 index dba8c7aee5..0000000000 --- a/vendor/github.com/coreos/fcct/base/v0_1/translate.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2019 Red Hat, Inc -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.) - -package v0_1 - -import ( - "net/url" - - "github.com/coreos/fcct/translate" - - "github.com/coreos/ignition/v2/config/v3_0/types" - "github.com/coreos/vcontext/path" - "github.com/vincent-petithory/dataurl" -) - -// ToIgn3_0 translates the config to an Ignition config. It also returns the set of translations -// it did so paths in the resultant config can be tracked back to their source in the source config. -func (c Config) ToIgn3_0() (types.Config, translate.TranslationSet, error) { - ret := types.Config{} - tr := translate.NewTranslator("yaml", "json") - tr.AddCustomTranslator(translateIgnition) - tr.AddCustomTranslator(translateFile) - tr.AddCustomTranslator(translateDirectory) - tr.AddCustomTranslator(translateLink) - translations := tr.Translate(&c, &ret) - return ret, translations, nil -} - -func translateIgnition(from Ignition) (to types.Ignition, tm translate.TranslationSet) { - tr := translate.NewTranslator("yaml", "json") - to.Version = types.MaxVersion.String() - tm = tr.Translate(&from.Config, &to.Config).Prefix("config") - tm.MergeP("security", tr.Translate(&from.Security, &to.Security)) - tm.MergeP("timeouts", tr.Translate(&from.Timeouts, &to.Timeouts)) - return -} - -func translateFile(from File) (to types.File, tm translate.TranslationSet) { - tr := translate.NewTranslator("yaml", "json") - tr.AddCustomTranslator(translateFileContents) - tm = tr.Translate(&from.Group, &to.Group).Prefix("group") - tm.MergeP("user", tr.Translate(&from.User, &to.User)) - tm.MergeP("append", tr.Translate(&from.Append, &to.Append)) - tm.MergeP("contents", tr.Translate(&from.Contents, &to.Contents)) - to.Overwrite = from.Overwrite - to.Path = from.Path - to.Mode = from.Mode - tm.AddIdentity("overwrite", "path", "mode") - return -} - -func translateFileContents(from FileContents) (to types.FileContents, tm translate.TranslationSet) { - tr := translate.NewTranslator("yaml", "json") - tm = tr.Translate(&from.Verification, &to.Verification).Prefix("verification") - to.Source = from.Source - to.Compression = from.Compression - tm.AddIdentity("source", "compression") - if from.Inline != nil { - src := (&url.URL{ - Scheme: "data", - Opaque: "," + dataurl.EscapeString(*from.Inline), - }).String() - to.Source = &src - tm.AddTranslation(path.New("yaml", "inline"), path.New("json", "source")) - } - return -} - -func translateDirectory(from Directory) (to types.Directory, tm translate.TranslationSet) { - tr := translate.NewTranslator("yaml", "json") - tm = tr.Translate(&from.Group, &to.Group).Prefix("group") - tm.MergeP("user", tr.Translate(&from.User, &to.User)) - to.Overwrite = from.Overwrite - to.Path = from.Path - to.Mode = from.Mode - tm.AddIdentity("overwrite", "path", "mode") - return -} - -func translateLink(from Link) (to types.Link, tm translate.TranslationSet) { - tr := translate.NewTranslator("yaml", "json") - tm = tr.Translate(&from.Group, &to.Group).Prefix("group") - tm.MergeP("user", tr.Translate(&from.User, &to.User)) - to.Target = from.Target - to.Hard = from.Hard - to.Overwrite = from.Overwrite - to.Path = from.Path - tm.AddIdentity("target", "hard", "overwrite", "path") - return -} diff --git a/vendor/github.com/coreos/fcct/translate/util.go b/vendor/github.com/coreos/fcct/translate/util.go deleted file mode 100644 index 6f2b6dd504..0000000000 --- a/vendor/github.com/coreos/fcct/translate/util.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package translate - -import ( - "reflect" - "strings" - - "github.com/coreos/ignition/v2/config/util" - "github.com/coreos/vcontext/path" -) - -// fieldName returns the name uses when (un)marshalling a field. t should be a reflect.Value of a struct, -// index is the field index, and tag is the struct tag used when (un)marshalling (e.g. "json" or "yaml") -func fieldName(t reflect.Value, index int, tag string) string { - f := t.Type().Field(index) - if tag == "" { - return f.Name - } - return strings.Split(f.Tag.Get(tag), ",")[0] -} - -func prefixPath(p path.ContextPath, prefix ...interface{}) path.ContextPath { - return path.New(p.Tag, prefix...).Append(p.Path...) -} - -func prefixPaths(ps []path.ContextPath, prefix ...interface{}) []path.ContextPath { - ret := []path.ContextPath{} - for _, p := range ps { - ret = append(ret, prefixPath(p, prefix...)) - } - return ret -} - -func getAllPaths(v reflect.Value, tag string) []path.ContextPath { - k := v.Kind() - t := v.Type() - switch { - case util.IsPrimitive(k): - return nil - case k == reflect.Ptr: - if v.IsNil() { - return nil - } - return getAllPaths(v.Elem(), tag) - case k == reflect.Slice: - ret := []path.ContextPath{} - for i := 0; i < v.Len(); i++ { - ret = append(ret, prefixPaths(getAllPaths(v.Index(i), tag), i)...) - } - return ret - case k == reflect.Struct: - ret := []path.ContextPath{} - for i := 0; i < t.NumField(); i++ { - name := fieldName(v, i, tag) - field := v.Field(i) - if t.Field(i).Anonymous { - ret = append(ret, getAllPaths(field, tag)...) - } else { - ret = append(ret, prefixPaths(getAllPaths(field, tag), name)...) - ret = append(ret, path.New(tag, name)) - } - } - return ret - default: - panic("Encountered types that are not the same when they should be. This is a bug, please file a report") - } -} diff --git a/vendor/github.com/coreos/vcontext/yaml/yaml.go b/vendor/github.com/coreos/vcontext/yaml/yaml.go new file mode 100644 index 0000000000..7e9048ada3 --- /dev/null +++ b/vendor/github.com/coreos/vcontext/yaml/yaml.go @@ -0,0 +1,82 @@ +// Copyright 2019 Red Hat, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.) + +package json + +import ( + "github.com/coreos/vcontext/tree" + + "gopkg.in/yaml.v3" +) + +func UnmarshalToContext(raw []byte) (tree.Node, error) { + var ast yaml.Node + if err := yaml.Unmarshal(raw, &ast); err != nil { + return nil, err + } + return fromYamlNode(ast), nil +} + +func fromYamlNode(n yaml.Node) tree.Node { + m := tree.Marker{ + StartP: &tree.Pos{ + Line: int64(n.Line), + Column: int64(n.Column), + }, + } + switch n.Kind { + case 0: + // empty + return nil + case yaml.DocumentNode: + if len(n.Content) == 0 { + return nil + } + return fromYamlNode(*n.Content[0]) + case yaml.MappingNode: + ret := tree.MapNode{ + Marker: m, + Children: make(map[string]tree.Node, len(n.Content)/2), + Keys: make(map[string]tree.Leaf, len(n.Content)/2), + } + // MappingNodes list keys and values like [k, v, k, v...] + for i := 0; i < len(n.Content); i += 2 { + key := *n.Content[i] + value := *n.Content[i+1] + ret.Keys[key.Value] = tree.Leaf{ + Marker: tree.Marker{ + StartP: &tree.Pos{ + Line: int64(key.Line), + Column: int64(key.Column), + }, + }, + } + ret.Children[key.Value] = fromYamlNode(value) + } + return ret + case yaml.SequenceNode: + ret := tree.SliceNode{ + Marker: m, + Children: make([]tree.Node, 0, len(n.Content)), + } + for _, child := range n.Content { + ret.Children = append(ret.Children, fromYamlNode(*child)) + } + return ret + default: // scalars and aliases + return tree.Leaf{ + Marker: m, + } + } +} diff --git a/vendor/github.com/spf13/pflag/flag.go b/vendor/github.com/spf13/pflag/flag.go index 24a5036e95..7c058de374 100644 --- a/vendor/github.com/spf13/pflag/flag.go +++ b/vendor/github.com/spf13/pflag/flag.go @@ -160,7 +160,7 @@ type FlagSet struct { args []string // arguments after flags argsLenAtDash int // len(args) when a '--' was located when parsing, or -1 if no -- errorHandling ErrorHandling - output io.Writer // nil means stderr; use out() accessor + output io.Writer // nil means stderr; use Output() accessor interspersed bool // allow interspersed option/non-option args normalizeNameFunc func(f *FlagSet, name string) NormalizedName @@ -255,13 +255,20 @@ func (f *FlagSet) normalizeFlagName(name string) NormalizedName { return n(f, name) } -func (f *FlagSet) out() io.Writer { +// Output returns the destination for usage and error messages. os.Stderr is returned if +// output was not set or was set to nil. +func (f *FlagSet) Output() io.Writer { if f.output == nil { return os.Stderr } return f.output } +// Name returns the name of the flag set. +func (f *FlagSet) Name() string { + return f.name +} + // SetOutput sets the destination for usage and error messages. // If output is nil, os.Stderr is used. func (f *FlagSet) SetOutput(output io.Writer) { @@ -358,7 +365,7 @@ func (f *FlagSet) ShorthandLookup(name string) *Flag { } if len(name) > 1 { msg := fmt.Sprintf("can not look up shorthand which is more than one ASCII character: %q", name) - fmt.Fprintf(f.out(), msg) + fmt.Fprintf(f.Output(), msg) panic(msg) } c := name[0] @@ -482,7 +489,7 @@ func (f *FlagSet) Set(name, value string) error { } if flag.Deprecated != "" { - fmt.Fprintf(f.out(), "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated) + fmt.Fprintf(f.Output(), "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated) } return nil } @@ -523,7 +530,7 @@ func Set(name, value string) error { // otherwise, the default values of all defined flags in the set. func (f *FlagSet) PrintDefaults() { usages := f.FlagUsages() - fmt.Fprint(f.out(), usages) + fmt.Fprint(f.Output(), usages) } // defaultIsZeroValue returns true if the default value for this flag represents @@ -758,7 +765,7 @@ func PrintDefaults() { // defaultUsage is the default function to print a usage message. func defaultUsage(f *FlagSet) { - fmt.Fprintf(f.out(), "Usage of %s:\n", f.name) + fmt.Fprintf(f.Output(), "Usage of %s:\n", f.name) f.PrintDefaults() } @@ -844,7 +851,7 @@ func (f *FlagSet) AddFlag(flag *Flag) { _, alreadyThere := f.formal[normalizedFlagName] if alreadyThere { msg := fmt.Sprintf("%s flag redefined: %s", f.name, flag.Name) - fmt.Fprintln(f.out(), msg) + fmt.Fprintln(f.Output(), msg) panic(msg) // Happens only if flags are declared with identical names } if f.formal == nil { @@ -860,7 +867,7 @@ func (f *FlagSet) AddFlag(flag *Flag) { } if len(flag.Shorthand) > 1 { msg := fmt.Sprintf("%q shorthand is more than one ASCII character", flag.Shorthand) - fmt.Fprintf(f.out(), msg) + fmt.Fprintf(f.Output(), msg) panic(msg) } if f.shorthands == nil { @@ -870,7 +877,7 @@ func (f *FlagSet) AddFlag(flag *Flag) { used, alreadyThere := f.shorthands[c] if alreadyThere { msg := fmt.Sprintf("unable to redefine %q shorthand in %q flagset: it's already used for %q flag", c, f.name, used.Name) - fmt.Fprintf(f.out(), msg) + fmt.Fprintf(f.Output(), msg) panic(msg) } f.shorthands[c] = flag @@ -909,7 +916,7 @@ func VarP(value Value, name, shorthand, usage string) { func (f *FlagSet) failf(format string, a ...interface{}) error { err := fmt.Errorf(format, a...) if f.errorHandling != ContinueOnError { - fmt.Fprintln(f.out(), err) + fmt.Fprintln(f.Output(), err) f.usage() } return err @@ -1060,7 +1067,7 @@ func (f *FlagSet) parseSingleShortArg(shorthands string, args []string, fn parse } if flag.ShorthandDeprecated != "" { - fmt.Fprintf(f.out(), "Flag shorthand -%s has been deprecated, %s\n", flag.Shorthand, flag.ShorthandDeprecated) + fmt.Fprintf(f.Output(), "Flag shorthand -%s has been deprecated, %s\n", flag.Shorthand, flag.ShorthandDeprecated) } err = fn(flag, value) diff --git a/vendor/github.com/spf13/pflag/ip.go b/vendor/github.com/spf13/pflag/ip.go index 3d414ba69f..06b8bcb572 100644 --- a/vendor/github.com/spf13/pflag/ip.go +++ b/vendor/github.com/spf13/pflag/ip.go @@ -16,6 +16,9 @@ func newIPValue(val net.IP, p *net.IP) *ipValue { func (i *ipValue) String() string { return net.IP(*i).String() } func (i *ipValue) Set(s string) error { + if s == "" { + return nil + } ip := net.ParseIP(strings.TrimSpace(s)) if ip == nil { return fmt.Errorf("failed to parse IP: %q", s) diff --git a/vendor/github.com/spf13/pflag/ipnet_slice.go b/vendor/github.com/spf13/pflag/ipnet_slice.go new file mode 100644 index 0000000000..6b541aa879 --- /dev/null +++ b/vendor/github.com/spf13/pflag/ipnet_slice.go @@ -0,0 +1,147 @@ +package pflag + +import ( + "fmt" + "io" + "net" + "strings" +) + +// -- ipNetSlice Value +type ipNetSliceValue struct { + value *[]net.IPNet + changed bool +} + +func newIPNetSliceValue(val []net.IPNet, p *[]net.IPNet) *ipNetSliceValue { + ipnsv := new(ipNetSliceValue) + ipnsv.value = p + *ipnsv.value = val + return ipnsv +} + +// Set converts, and assigns, the comma-separated IPNet argument string representation as the []net.IPNet value of this flag. +// If Set is called on a flag that already has a []net.IPNet assigned, the newly converted values will be appended. +func (s *ipNetSliceValue) Set(val string) error { + + // remove all quote characters + rmQuote := strings.NewReplacer(`"`, "", `'`, "", "`", "") + + // read flag arguments with CSV parser + ipNetStrSlice, err := readAsCSV(rmQuote.Replace(val)) + if err != nil && err != io.EOF { + return err + } + + // parse ip values into slice + out := make([]net.IPNet, 0, len(ipNetStrSlice)) + for _, ipNetStr := range ipNetStrSlice { + _, n, err := net.ParseCIDR(strings.TrimSpace(ipNetStr)) + if err != nil { + return fmt.Errorf("invalid string being converted to CIDR: %s", ipNetStr) + } + out = append(out, *n) + } + + if !s.changed { + *s.value = out + } else { + *s.value = append(*s.value, out...) + } + + s.changed = true + + return nil +} + +// Type returns a string that uniquely represents this flag's type. +func (s *ipNetSliceValue) Type() string { + return "ipNetSlice" +} + +// String defines a "native" format for this net.IPNet slice flag value. +func (s *ipNetSliceValue) String() string { + + ipNetStrSlice := make([]string, len(*s.value)) + for i, n := range *s.value { + ipNetStrSlice[i] = n.String() + } + + out, _ := writeAsCSV(ipNetStrSlice) + return "[" + out + "]" +} + +func ipNetSliceConv(val string) (interface{}, error) { + val = strings.Trim(val, "[]") + // Emtpy string would cause a slice with one (empty) entry + if len(val) == 0 { + return []net.IPNet{}, nil + } + ss := strings.Split(val, ",") + out := make([]net.IPNet, len(ss)) + for i, sval := range ss { + _, n, err := net.ParseCIDR(strings.TrimSpace(sval)) + if err != nil { + return nil, fmt.Errorf("invalid string being converted to CIDR: %s", sval) + } + out[i] = *n + } + return out, nil +} + +// GetIPNetSlice returns the []net.IPNet value of a flag with the given name +func (f *FlagSet) GetIPNetSlice(name string) ([]net.IPNet, error) { + val, err := f.getFlagType(name, "ipNetSlice", ipNetSliceConv) + if err != nil { + return []net.IPNet{}, err + } + return val.([]net.IPNet), nil +} + +// IPNetSliceVar defines a ipNetSlice flag with specified name, default value, and usage string. +// The argument p points to a []net.IPNet variable in which to store the value of the flag. +func (f *FlagSet) IPNetSliceVar(p *[]net.IPNet, name string, value []net.IPNet, usage string) { + f.VarP(newIPNetSliceValue(value, p), name, "", usage) +} + +// IPNetSliceVarP is like IPNetSliceVar, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) IPNetSliceVarP(p *[]net.IPNet, name, shorthand string, value []net.IPNet, usage string) { + f.VarP(newIPNetSliceValue(value, p), name, shorthand, usage) +} + +// IPNetSliceVar defines a []net.IPNet flag with specified name, default value, and usage string. +// The argument p points to a []net.IPNet variable in which to store the value of the flag. +func IPNetSliceVar(p *[]net.IPNet, name string, value []net.IPNet, usage string) { + CommandLine.VarP(newIPNetSliceValue(value, p), name, "", usage) +} + +// IPNetSliceVarP is like IPNetSliceVar, but accepts a shorthand letter that can be used after a single dash. +func IPNetSliceVarP(p *[]net.IPNet, name, shorthand string, value []net.IPNet, usage string) { + CommandLine.VarP(newIPNetSliceValue(value, p), name, shorthand, usage) +} + +// IPNetSlice defines a []net.IPNet flag with specified name, default value, and usage string. +// The return value is the address of a []net.IPNet variable that stores the value of that flag. +func (f *FlagSet) IPNetSlice(name string, value []net.IPNet, usage string) *[]net.IPNet { + p := []net.IPNet{} + f.IPNetSliceVarP(&p, name, "", value, usage) + return &p +} + +// IPNetSliceP is like IPNetSlice, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) IPNetSliceP(name, shorthand string, value []net.IPNet, usage string) *[]net.IPNet { + p := []net.IPNet{} + f.IPNetSliceVarP(&p, name, shorthand, value, usage) + return &p +} + +// IPNetSlice defines a []net.IPNet flag with specified name, default value, and usage string. +// The return value is the address of a []net.IP variable that stores the value of the flag. +func IPNetSlice(name string, value []net.IPNet, usage string) *[]net.IPNet { + return CommandLine.IPNetSliceP(name, "", value, usage) +} + +// IPNetSliceP is like IPNetSlice, but accepts a shorthand letter that can be used after a single dash. +func IPNetSliceP(name, shorthand string, value []net.IPNet, usage string) *[]net.IPNet { + return CommandLine.IPNetSliceP(name, shorthand, value, usage) +} diff --git a/vendor/github.com/spf13/pflag/string_array.go b/vendor/github.com/spf13/pflag/string_array.go index 4894af8180..d1ff0a96ba 100644 --- a/vendor/github.com/spf13/pflag/string_array.go +++ b/vendor/github.com/spf13/pflag/string_array.go @@ -31,11 +31,7 @@ func (s *stringArrayValue) Append(val string) error { func (s *stringArrayValue) Replace(val []string) error { out := make([]string, len(val)) for i, d := range val { - var err error out[i] = d - if err != nil { - return err - } } *s.value = out return nil diff --git a/vendor/modules.txt b/vendor/modules.txt index d501bd1a51..e5a911a58b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -145,10 +145,14 @@ github.com/containers/storage/pkg/reexec github.com/containers/storage/pkg/stringid github.com/containers/storage/pkg/system github.com/containers/storage/pkg/unshare -# github.com/coreos/fcct v0.5.0 -## explicit; go 1.12 -github.com/coreos/fcct/base/v0_1 -github.com/coreos/fcct/translate +# github.com/coreos/butane v0.14.0 +## explicit; go 1.15 +github.com/coreos/butane/base/util +github.com/coreos/butane/base/v0_3 +github.com/coreos/butane/config/common +github.com/coreos/butane/config/fcos/v1_2 +github.com/coreos/butane/config/util +github.com/coreos/butane/translate # github.com/coreos/go-json v0.0.0-20211020211907-c63f628265de ## explicit; go 1.15 github.com/coreos/go-json @@ -210,6 +214,7 @@ github.com/coreos/vcontext/path github.com/coreos/vcontext/report github.com/coreos/vcontext/tree github.com/coreos/vcontext/validate +github.com/coreos/vcontext/yaml # github.com/daixiang0/gci v0.2.9 ## explicit; go 1.14 github.com/daixiang0/gci/pkg/gci @@ -875,7 +880,7 @@ github.com/spf13/cobra # github.com/spf13/jwalterweatherman v1.1.0 ## explicit github.com/spf13/jwalterweatherman -# github.com/spf13/pflag v1.0.5 +# github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace ## explicit; go 1.12 github.com/spf13/pflag # github.com/spf13/viper v1.8.1