From b4990760e0a7db73dabcbcd2757fe2890008feb6 Mon Sep 17 00:00:00 2001 From: steiler Date: Thu, 7 Apr 2022 14:34:05 +0200 Subject: [PATCH 1/7] adding env-files to kind and node topology definition --- clab/config.go | 10 +++++++++- types/node_definition.go | 9 +++++++++ types/topology.go | 10 ++++++++++ utils/env.go | 39 +++++++++++++++++++++++++++++++++++++++ utils/file.go | 22 ++++++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) diff --git a/clab/config.go b/clab/config.go index 446c2fd19..dbfcf94a9 100644 --- a/clab/config.go +++ b/clab/config.go @@ -260,9 +260,17 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx // Extras Extras: c.Config.Topology.GetNodeExtras(nodeName), } + var err error + + // Load content of the EnvVarFiles + envFileContent, err := utils.LoadEnvVarFiles(c.Config.Topology.GetNodeEnvFiles(nodeName)) + if err != nil { + return nil, err + } + // Merge EnvVarFiles content and the existing env variable + nodeCfg.Env = utils.MergeStringMaps(envFileContent, nodeCfg.Env) log.Debugf("node config: %+v", nodeCfg) - var err error // initialize config p, err := c.Config.Topology.GetNodeStartupConfig(nodeCfg.ShortName) if err != nil { diff --git a/types/node_definition.go b/types/node_definition.go index 8cbb7cad4..c0dffb3f1 100644 --- a/types/node_definition.go +++ b/types/node_definition.go @@ -37,6 +37,8 @@ type NodeDefinition struct { Publish []string `yaml:"publish,omitempty"` // environment variables Env map[string]string `yaml:"env,omitempty"` + // external file containing environment variables + EnvFiles []string `yaml:"env-files,omitempty"` // linux user used in a container User string `yaml:"user,omitempty"` // container labels @@ -186,6 +188,13 @@ func (n *NodeDefinition) GetEnv() map[string]string { return n.Env } +func (n *NodeDefinition) GetEnvFiles() []string { + if n == nil { + return nil + } + return n.EnvFiles +} + func (n *NodeDefinition) GetUser() string { if n == nil { return "" diff --git a/types/topology.go b/types/topology.go index 60cb04265..18bb9710e 100644 --- a/types/topology.go +++ b/types/topology.go @@ -105,6 +105,16 @@ func (t *Topology) GetNodeEnv(name string) map[string]string { return nil } +func (t *Topology) GetNodeEnvFiles(name string) []string { + if ndef, ok := t.Nodes[name]; ok { + return utils.MergeStringSlices( + utils.MergeStringSlices(t.GetDefaults().GetEnvFiles(), + t.GetKind(t.GetNodeKind(name)).GetEnvFiles()), + ndef.GetEnvFiles()) + } + return nil +} + func (t *Topology) GetNodePublish(name string) []string { if ndef, ok := t.Nodes[name]; ok { if len(ndef.GetPublish()) > 0 { diff --git a/utils/env.go b/utils/env.go index d00df030e..fcf3e92ad 100644 --- a/utils/env.go +++ b/utils/env.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "reflect" + "strings" ) // convertEnvs convert env variables passed as a map to a list of them @@ -87,6 +88,44 @@ func StringInSlice(slice []string, val string) (int, bool) { return -1, false } +// load EnvVars from files +func LoadEnvVarFiles(files []string) (map[string]string, error) { + result := map[string]string{} + // iterate over filesnames + for _, file := range files { + // read file content + lines, err := ReadFileLines(file) + if err != nil { + return nil, err + } + // disect the string slices into maps with environment keys and values + envMap, err := envVarLineToMap(lines) + if err != nil { + return nil, err + } + // merge the actual file content with the overall result + result = MergeStringMaps(result, envMap) + } + return result, nil +} + +// envVarLineToMap splits the env variable definiton lines into key and value +func envVarLineToMap(lines []string) (map[string]string, error) { + result := map[string]string{} + // iterate over lines + for _, line := range lines { + // split on the = sign + splitSlice := strings.Split(line, "=") + // we expect to see at least two elements in the slice (one = sign present) + if len(splitSlice) < 2 { + fmt.Errorf("Issue with format of Env file line '%s'", line) + } + // take the first element as the key and join the rest back with = as the value + result[splitSlice[0]] = strings.Join(splitSlice[1:], "=") + } + return result, nil +} + // MergeStringSlices merges string slices with duplicates removed func MergeStringSlices(ss ...[]string) []string { res := make([]string, 0) diff --git a/utils/file.go b/utils/file.go index e11d2d60e..f020c4391 100644 --- a/utils/file.go +++ b/utils/file.go @@ -5,10 +5,12 @@ package utils import ( + "bufio" "errors" "fmt" "io" "io/ioutil" + "log" "net/http" "os" "path/filepath" @@ -147,6 +149,26 @@ func ReadFileContent(file string) ([]byte, error) { return b, err } +func ReadFileLines(file string) ([]string, error) { + // check file exists + if !FileExists(file) { + return nil, fmt.Errorf("%w: %s", errFileNotExist, file) + } + content, err := os.Open(file) + if err != nil { + log.Fatalf("failed opening file: %s", err) + } + + scanner := bufio.NewScanner(content) + scanner.Split(bufio.ScanLines) + var result []string + + for scanner.Scan() { + result = append(result, scanner.Text()) + } + return result, nil +} + // ExpandHome expands `~` char in the path to home path of a current user in provided path p. func ExpandHome(p string) string { userPath, _ := os.UserHomeDir() From 6be5b50c113a2e7a6bbad5a57392e2bb297201b5 Mon Sep 17 00:00:00 2001 From: steiler Date: Thu, 7 Apr 2022 14:57:13 +0200 Subject: [PATCH 2/7] make the topology .clab.yml folder the base path for the env-file reference --- clab/config.go | 2 +- utils/env.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/clab/config.go b/clab/config.go index dbfcf94a9..3fa86748f 100644 --- a/clab/config.go +++ b/clab/config.go @@ -263,7 +263,7 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx var err error // Load content of the EnvVarFiles - envFileContent, err := utils.LoadEnvVarFiles(c.Config.Topology.GetNodeEnvFiles(nodeName)) + envFileContent, err := utils.LoadEnvVarFiles(c.TopoFile.dir, c.Config.Topology.GetNodeEnvFiles(nodeName)) if err != nil { return nil, err } diff --git a/utils/env.go b/utils/env.go index fcf3e92ad..a4ddb4a08 100644 --- a/utils/env.go +++ b/utils/env.go @@ -7,6 +7,7 @@ package utils import ( "fmt" "os" + "path/filepath" "reflect" "strings" ) @@ -89,10 +90,15 @@ func StringInSlice(slice []string, val string) (int, bool) { } // load EnvVars from files -func LoadEnvVarFiles(files []string) (map[string]string, error) { +func LoadEnvVarFiles(baseDir string, files []string) (map[string]string, error) { result := map[string]string{} // iterate over filesnames for _, file := range files { + // if not a root based path, we take it relative from the xyz.clab.yml file + if file[0] != '/' { + file = filepath.Join(baseDir, file) + } + // read file content lines, err := ReadFileLines(file) if err != nil { @@ -118,7 +124,7 @@ func envVarLineToMap(lines []string) (map[string]string, error) { splitSlice := strings.Split(line, "=") // we expect to see at least two elements in the slice (one = sign present) if len(splitSlice) < 2 { - fmt.Errorf("Issue with format of Env file line '%s'", line) + return nil, fmt.Errorf("issue with format of env file line '%s'", line) } // take the first element as the key and join the rest back with = as the value result[splitSlice[0]] = strings.Join(splitSlice[1:], "=") From b175333e51b987bbee21f0b662772f79f14da4cb Mon Sep 17 00:00:00 2001 From: steiler Date: Fri, 8 Apr 2022 08:46:49 +0200 Subject: [PATCH 3/7] removing log.Fatalf from non-main() --- utils/file.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/file.go b/utils/file.go index f020c4391..d7138ad7a 100644 --- a/utils/file.go +++ b/utils/file.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "os" "path/filepath" @@ -156,7 +155,7 @@ func ReadFileLines(file string) ([]string, error) { } content, err := os.Open(file) if err != nil { - log.Fatalf("failed opening file: %s", err) + return nil, err } scanner := bufio.NewScanner(content) From 72e01777028ed66f9e9487c9b590800d6879ec8d Mon Sep 17 00:00:00 2001 From: steiler Date: Mon, 11 Apr 2022 12:33:05 +0200 Subject: [PATCH 4/7] changed implementation to use godotenv package and added doc --- docs/manual/nodes.md | 23 +++++++++++++++++++++++ utils/env.go | 41 +++++++++++------------------------------ 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/docs/manual/nodes.md b/docs/manual/nodes.md index 841da3f17..9b4e1c93e 100644 --- a/docs/manual/nodes.md +++ b/docs/manual/nodes.md @@ -202,6 +202,29 @@ topology: You can also specify a magic ENV VAR - `__IMPORT_ENVS: true` - which will import all environment variables defined in your shell to the relevant topology level. +### env-files +To add environment variables defined in files to a node use the `env-files` container that can be added at `defaults`, `kind` and `node` levels. + +The variable definitions of all the specified files are merged. More specific definitions (default -> kind -> node) will overwrite less specific. +Files can either be specified with their absolute path, but also with a relative path. The base path for the relative path resolution is the directory that holds the clab.yml topology definition + +```yaml +topology: + defaults: + env-files: + - envfiles/defaults + - /home/user/clab/default-env + kinds: + srl: + env-files: + - envfiles/common + - envfiles/spines + nodes: + node1: + env-files: + - /home/user/somefile +``` + ### user To set a user which will be used to run a containerized process use the `user` configuration option. Can be defined at `node`, `kind` and `global` levels. diff --git a/utils/env.go b/utils/env.go index a4ddb4a08..5a52b3bf8 100644 --- a/utils/env.go +++ b/utils/env.go @@ -9,7 +9,8 @@ import ( "os" "path/filepath" "reflect" - "strings" + + "github.com/joho/godotenv" ) // convertEnvs convert env variables passed as a map to a list of them @@ -91,45 +92,25 @@ func StringInSlice(slice []string, val string) (int, bool) { // load EnvVars from files func LoadEnvVarFiles(baseDir string, files []string) (map[string]string, error) { - result := map[string]string{} + resolvedfiles := []string{} // iterate over filesnames for _, file := range files { // if not a root based path, we take it relative from the xyz.clab.yml file if file[0] != '/' { file = filepath.Join(baseDir, file) } - - // read file content - lines, err := ReadFileLines(file) - if err != nil { - return nil, err - } - // disect the string slices into maps with environment keys and values - envMap, err := envVarLineToMap(lines) - if err != nil { - return nil, err - } - // merge the actual file content with the overall result - result = MergeStringMaps(result, envMap) + resolvedfiles = append(resolvedfiles, file) + } + if len(resolvedfiles) == 0 { + return map[string]string{}, nil } - return result, nil -} -// envVarLineToMap splits the env variable definiton lines into key and value -func envVarLineToMap(lines []string) (map[string]string, error) { - result := map[string]string{} - // iterate over lines - for _, line := range lines { - // split on the = sign - splitSlice := strings.Split(line, "=") - // we expect to see at least two elements in the slice (one = sign present) - if len(splitSlice) < 2 { - return nil, fmt.Errorf("issue with format of env file line '%s'", line) - } - // take the first element as the key and join the rest back with = as the value - result[splitSlice[0]] = strings.Join(splitSlice[1:], "=") + result, err := godotenv.Read(resolvedfiles...) + if err != nil { + return nil, err } return result, nil + } // MergeStringSlices merges string slices with duplicates removed From 4325ef73c3715154ee3b05890f7342691657b009 Mon Sep 17 00:00:00 2001 From: steiler Date: Mon, 11 Apr 2022 15:20:03 +0200 Subject: [PATCH 5/7] utilizing utils.ResolvePath --- utils/env.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/utils/env.go b/utils/env.go index 5a52b3bf8..08107ecbf 100644 --- a/utils/env.go +++ b/utils/env.go @@ -7,7 +7,6 @@ package utils import ( "fmt" "os" - "path/filepath" "reflect" "github.com/joho/godotenv" @@ -90,27 +89,27 @@ func StringInSlice(slice []string, val string) (int, bool) { return -1, false } -// load EnvVars from files -func LoadEnvVarFiles(baseDir string, files []string) (map[string]string, error) { - resolvedfiles := []string{} - // iterate over filesnames +// LoadEnvVarFiles load EnvVars from the given files, resolving relative paths +func LoadEnvVarFiles(basefolder string, files []string) (map[string]string, error) { + resolvedPaths := []string{} + // resolve given paths, relative (to topology definition file) for _, file := range files { - // if not a root based path, we take it relative from the xyz.clab.yml file - if file[0] != '/' { - file = filepath.Join(baseDir, file) + resolved := ResolvePath(file, basefolder) + if !FileExists(resolved) { + return nil, fmt.Errorf("env-file %s not found (path resolved to %s)", file, resolved) } - resolvedfiles = append(resolvedfiles, file) + resolvedPaths = append(resolvedPaths, resolved) } - if len(resolvedfiles) == 0 { + + if len(resolvedPaths) == 0 { return map[string]string{}, nil } - result, err := godotenv.Read(resolvedfiles...) + result, err := godotenv.Read(resolvedPaths...) if err != nil { return nil, err } return result, nil - } // MergeStringSlices merges string slices with duplicates removed From 3ce0ad6eea14ccf0498895acfce30926ca003661 Mon Sep 17 00:00:00 2001 From: steiler Date: Tue, 12 Apr 2022 14:16:46 +0200 Subject: [PATCH 6/7] env-file unittest added --- clab/config_test.go | 51 +++++++++++++++++++++++++++++++++++++++ clab/test_data/envfile1 | 1 + clab/test_data/envfile2 | 2 ++ clab/test_data/topo10.yml | 20 +++++++++++++++ 4 files changed, 74 insertions(+) create mode 100644 clab/test_data/envfile1 create mode 100644 clab/test_data/envfile2 create mode 100644 clab/test_data/topo10.yml diff --git a/clab/config_test.go b/clab/config_test.go index 2f8013ab0..aaa88d0f8 100644 --- a/clab/config_test.go +++ b/clab/config_test.go @@ -470,3 +470,54 @@ func TestVerifyRootNetnsInterfaceUniqueness(t *testing.T) { t.Logf("error: %v", err) } + +func TestEnvFileInit(t *testing.T) { + tests := map[string]struct { + got string + node string + want map[string]string + }{ + "env-file_defined_at_node_and_default_1": { + got: "test_data/topo10.yml", + node: "node1", + want: map[string]string{ + "env1": "val1", + "env2": "val2", + "ENVFILE1": "SOMEOTHERDATA", + "ENVFILE2": "THISANDTHAT", + }, + }, + "env-file_defined_at_node_and_default_2": { + got: "test_data/topo10.yml", + node: "node2", + want: map[string]string{ + "ENVFILE1": "SOMEENVVARDATA", + "ENVFILE2": "THISANDTHAT", + }, + }, + } + + teardownTestCase := setupTestCase(t) + defer teardownTestCase(t) + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + opts := []ClabOption{ + WithTopoFile(tc.got, ""), + } + c, err := NewContainerLab(opts...) + if err != nil { + t.Fatal(err) + } + + env := c.Nodes[tc.node].Config().Env + // check all the want key/values are there + for k, v := range tc.want { + //check keys defined in tc.want exist and values are equal + if val, exists := env[k]; !(exists && val == v) { + t.Fatalf("wanted %q to be contained in env, but got %q", tc.want, env) + } + } + }) + } +} diff --git a/clab/test_data/envfile1 b/clab/test_data/envfile1 new file mode 100644 index 000000000..d309780b2 --- /dev/null +++ b/clab/test_data/envfile1 @@ -0,0 +1 @@ +ENVFILE1=SOMEENVVARDATA \ No newline at end of file diff --git a/clab/test_data/envfile2 b/clab/test_data/envfile2 new file mode 100644 index 000000000..617b5c59d --- /dev/null +++ b/clab/test_data/envfile2 @@ -0,0 +1,2 @@ +ENVFILE1=SOMEOTHERDATA +ENVFILE2=THISANDTHAT \ No newline at end of file diff --git a/clab/test_data/topo10.yml b/clab/test_data/topo10.yml new file mode 100644 index 000000000..f168d3aaf --- /dev/null +++ b/clab/test_data/topo10.yml @@ -0,0 +1,20 @@ +name: topo10 +topology: + defaults: + env-files: + - envfile2 + nodes: + node1: + kind: linux + env: + env1: val1 + env2: val2 + mgmt_ipv4: 172.100.100.11 + node2: + kind: linux + mgmt_ipv4: 172.100.100.12 + labels: + node-label: value + env-files: + - envfile1 + From f41b242b8b7d5092d10f1ee3888079714df7dbcd Mon Sep 17 00:00:00 2001 From: Roman Dodin Date: Tue, 12 Apr 2022 16:47:09 +0200 Subject: [PATCH 7/7] brushed doc section --- docs/manual/nodes.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/manual/nodes.md b/docs/manual/nodes.md index 9b4e1c93e..073344622 100644 --- a/docs/manual/nodes.md +++ b/docs/manual/nodes.md @@ -203,10 +203,11 @@ topology: You can also specify a magic ENV VAR - `__IMPORT_ENVS: true` - which will import all environment variables defined in your shell to the relevant topology level. ### env-files -To add environment variables defined in files to a node use the `env-files` container that can be added at `defaults`, `kind` and `node` levels. +To add environment variables defined in a file use the `env-files` property that can be defined at `defaults`, `kind` and `node` levels. -The variable definitions of all the specified files are merged. More specific definitions (default -> kind -> node) will overwrite less specific. -Files can either be specified with their absolute path, but also with a relative path. The base path for the relative path resolution is the directory that holds the clab.yml topology definition +The variable defined in the files are merged across all of them wtit more specific definitions overwriting less specific. Node level is the most specific one. + +Files can either be specified with their absolute path or a relative path. The base path for the relative path resolution is the directory that holds the topology definition file. ```yaml topology: @@ -218,7 +219,7 @@ topology: srl: env-files: - envfiles/common - - envfiles/spines + - ~/spines nodes: node1: env-files: