Merge pull request #178 from hashicorp/gh-157

policy: add new file source for loading policies from disk.

agent/agent.go

@@ -9,7 +9,8 @@ import (
 	nomadHelper "github.com/hashicorp/nomad-autoscaler/helper/nomad"
 	"github.com/hashicorp/nomad-autoscaler/plugins/manager"
 	"github.com/hashicorp/nomad-autoscaler/policy"
-	nomadpolicy "github.com/hashicorp/nomad-autoscaler/policy/nomad"
+	filePolicy "github.com/hashicorp/nomad-autoscaler/policy/file"
+	nomadPolicy "github.com/hashicorp/nomad-autoscaler/policy/nomad"
 	"github.com/hashicorp/nomad/api"
 )
 
@@ -51,14 +52,7 @@ func (a *Agent) Run(ctx context.Context) error {
 	a.healthServer = healthServer
 	go a.healthServer.run()
 
-	sourceConfig := &nomadpolicy.SourceConfig{
-		DefaultCooldown:           a.config.Policy.DefaultCooldown,
-		DefaultEvaluationInterval: a.config.DefaultEvaluationInterval,
-	}
-	source := nomadpolicy.NewNomadSource(a.logger, a.nomadClient, sourceConfig)
-	a.policyManager = policy.NewManager(a.logger, source, a.pluginManager)
-
-	policyEvalCh := make(chan *policy.Evaluation, 10)
+	policyEvalCh := a.setupPolicyManager()
 	go a.policyManager.Run(ctx, policyEvalCh)
 
 	for {
@@ -73,6 +67,28 @@ func (a *Agent) Run(ctx context.Context) error {
 	}
 }
 
+func (a *Agent) setupPolicyManager() chan *policy.Evaluation {
+	sourceConfig := &policy.ConfigDefaults{
+		DefaultCooldown:           a.config.Policy.DefaultCooldown,
+		DefaultEvaluationInterval: a.config.DefaultEvaluationInterval,
+	}
+
+	// Setup our initial default policy source which is Nomad.
+	sources := map[policy.SourceName]policy.Source{
+		policy.SourceNameNomad: nomadPolicy.NewNomadSource(a.logger, a.nomadClient, sourceConfig),
+	}
+
+	// If the operators has configured a scaling policy directory to read from
+	// then setup the file source.
+	if a.config.Policy.Dir != "" {
+		sources[policy.SourceNameFile] = filePolicy.NewFileSource(a.logger, sourceConfig, a.config.Policy.Dir, make(chan bool))
+	}
+
+	a.policyManager = policy.NewManager(a.logger, sources, a.pluginManager)
+
+	return make(chan *policy.Evaluation, 10)
+}
+
 func (a *Agent) stop() {
 	// Stop the health server.
 	if a.healthServer != nil {

agent/config/config.go

@@ -2,14 +2,13 @@ package config
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/hashicorp/hcl/v2/hclsimple"
+	"github.com/hashicorp/nomad-autoscaler/helper/file"
 	"github.com/hashicorp/nomad-autoscaler/plugins"
 	"github.com/mitchellh/copystructure"
 )
@@ -433,50 +432,10 @@ func Load(path string) (*Agent, error) {
 // loadDir loads all the configurations in the given directory in alphabetical
 // order.
 func loadDir(dir string) (*Agent, error) {
-	f, err := os.Open(dir)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
 
-	fi, err := f.Stat()
+	files, err := file.GetFileListFromDir(dir, ".hcl", ".json")
 	if err != nil {
-		return nil, err
-	}
-	if !fi.IsDir() {
-		return nil, fmt.Errorf("configuration path must be a directory: %s", dir)
-	}
-
-	var files []string
-	err = nil
-	for err != io.EOF {
-		var fis []os.FileInfo
-		fis, err = f.Readdir(128)
-		if err != nil && err != io.EOF {
-			return nil, err
-		}
-
-		for _, fi := range fis {
-			// Ignore directories
-			if fi.IsDir() {
-				continue
-			}
-
-			// Only care about files that are valid to load.
-			name := fi.Name()
-			skip := true
-			if strings.HasSuffix(name, ".hcl") {
-				skip = false
-			} else if strings.HasSuffix(name, ".json") {
-				skip = false
-			}
-			if skip || isTemporaryFile(name) {
-				continue
-			}
-
-			path := filepath.Join(dir, name)
-			files = append(files, path)
-		}
+		return nil, fmt.Errorf("failed to load config directory: %v", err)
 	}
 
 	// Fast-path if we have no files
@@ -504,11 +463,3 @@ func loadDir(dir string) (*Agent, error) {
 
 	return result, nil
 }
-
-// isTemporaryFile returns true or false depending on whether the provided file
-// name is a temporary file for the following editors: emacs or vim.
-func isTemporaryFile(name string) bool {
-	return strings.HasSuffix(name, "~") || // vim
-		strings.HasPrefix(name, ".#") || // emacs
-		(strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#")) // emacs
-}

agent/config/config_test.go

@@ -270,42 +270,3 @@ func TestAgent_loadDir(t *testing.T) {
 	assert.Equal(t, time.Duration(10000000000), cfg.DefaultEvaluationInterval)
 	assert.Equal(t, "/opt/nomad-autoscaler/plugins", cfg.PluginDir)
 }
-
-func Test_isTemporaryFile(t *testing.T) {
-	testCases := []struct {
-		testName       string
-		inputName      string
-		expectedReturn bool
-	}{
-		{
-			testName:       "vim temp input file",
-			inputName:      "config.hcl~",
-			expectedReturn: true,
-		},
-		{
-			testName:       "emacs temp input file 1",
-			inputName:      ".#config.hcl",
-			expectedReturn: true,
-		},
-		{
-			testName:       "emacs temp input file 2",
-			inputName:      "#config.hcl#",
-			expectedReturn: true,
-		},
-		{
-			testName:       "non-temp HCL config input file",
-			inputName:      "config.hcl",
-			expectedReturn: false,
-		},
-		{
-			testName:       "non-temp JSON config input file",
-			inputName:      "config.json",
-			expectedReturn: false,
-		},
-	}
-
-	for _, tc := range testCases {
-		actualOutput := isTemporaryFile(tc.inputName)
-		assert.Equal(t, tc.expectedReturn, actualOutput, tc.testName)
-	}
-}

helper/file/dir.go

@@ -0,0 +1,65 @@
+package file
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"path/filepath"
+)
+
+func GetFileListFromDir(dir string, suffixes ...string) ([]string, error) {
+
+	f, err := os.Open(dir)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("configuration path must be a directory: %s", dir)
+	}
+
+	var files []string
+	err = nil
+	for err != io.EOF {
+		var fis []os.FileInfo
+		fis, err = f.Readdir(128)
+		if err != nil && err != io.EOF {
+			return nil, err
+		}
+
+		for _, fi := range fis {
+			// Ignore directories
+			if fi.IsDir() {
+				continue
+			}
+
+			// Only care about files that are valid to load.
+			name := fi.Name()
+
+			if suffixes != nil {
+				if !fileHasSuffix(name, suffixes) || IsTemporaryFile(name) {
+					continue
+				}
+				path := filepath.Join(dir, name)
+				files = append(files, path)
+			}
+		}
+	}
+	return files, nil
+}
+
+func fileHasSuffix(file string, suffixes []string) bool {
+	for _, suffix := range suffixes {
+		if strings.HasSuffix(file, suffix) {
+			return true
+		}
+	}
+	return false
+}

helper/file/temp.go

@@ -0,0 +1,11 @@
+package file
+
+import "strings"
+
+// IsTemporaryFile returns true or false depending on whether the provided file
+// name is a temporary file for the following editors: emacs or vim.
+func IsTemporaryFile(name string) bool {
+	return strings.HasSuffix(name, "~") || // vim
+		strings.HasPrefix(name, ".#") || // emacs
+		(strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#")) // emacs
+}

helper/file/temp_test.go

@@ -0,0 +1,46 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_IsTemporaryFile(t *testing.T) {
+	testCases := []struct {
+		testName       string
+		inputName      string
+		expectedReturn bool
+	}{
+		{
+			testName:       "vim temp input file",
+			inputName:      "config.hcl~",
+			expectedReturn: true,
+		},
+		{
+			testName:       "emacs temp input file 1",
+			inputName:      ".#config.hcl",
+			expectedReturn: true,
+		},
+		{
+			testName:       "emacs temp input file 2",
+			inputName:      "#config.hcl#",
+			expectedReturn: true,
+		},
+		{
+			testName:       "non-temp HCL config input file",
+			inputName:      "config.hcl",
+			expectedReturn: false,
+		},
+		{
+			testName:       "non-temp JSON config input file",
+			inputName:      "config.json",
+			expectedReturn: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		actualOutput := IsTemporaryFile(tc.inputName)
+		assert.Equal(t, tc.expectedReturn, actualOutput, tc.testName)
+	}
+}

helper/scaleutils/filter_test.go

@@ -144,7 +144,6 @@ func Test_filterByClass(t *testing.T) {
 			assert.Equal(t, tc.expectedOutputList, actualOutput, tc.name)
 		})
 	}
-
 }
 
 func Test_awsNodeIDMap(t *testing.T) {

helper/uuid/uuid.go

@@ -0,0 +1,21 @@
+package uuid
+
+import (
+	crand "crypto/rand"
+	"fmt"
+)
+
+// Generate is used to generate a random UUID.
+func Generate() string {
+	buf := make([]byte, 16)
+	if _, err := crand.Read(buf); err != nil {
+		panic(fmt.Errorf("failed to read random bytes: %v", err))
+	}
+
+	return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x",
+		buf[0:4],
+		buf[4:6],
+		buf[6:8],
+		buf[8:10],
+		buf[10:16])
+}

helper/uuid/uuid_test.go

@@ -0,0 +1,41 @@
+package uuid
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerate(t *testing.T) {
+	testCases := []struct {
+		count int
+		name  string
+	}{
+		{count: 100, name: "generate 100 unique uuids"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			stored := make(map[string]interface{})
+
+			for i := 0; i < 100; i++ {
+
+				newUUID := Generate()
+
+				// Check the UUID matches the expected regex.
+				matched, err := regexp.MatchString("[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}", newUUID)
+				assert.True(t, matched, tc.name)
+				assert.Nil(t, err, tc.name)
+
+				// Ensure we have not seen the UUID before. Then store the new
+				// UUID.
+				val, ok := stored[newUUID]
+				assert.False(t, ok, tc.name)
+				assert.Nil(t, val, tc.name)
+				stored[newUUID] = nil
+			}
+		})
+	}
+}