Enable user defined port and proto for metrics http listener

Dockerfile

@@ -8,7 +8,7 @@
 # OF ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
-FROM alpine:3.5
+FROM alpine:3.8
 LABEL maintainer="Stegen Smith <matthsmi@adobe.com>"
 
 RUN apk update && apk add bash ca-certificates

Makefile

@@ -21,16 +21,18 @@ GO:=go
 pkgs=$(shell $(GO) list ./... | egrep -v "(vendor)")
 
 export DOCKERHUB_USER=$(shell echo "$$DOCKERHUB_USER")
-export BUTLER_VERSION=1.2.3
+export BUTLER_VERSION=1.3.0
 export VERSION=v$(BUTLER_VERSION)
 
 default: ci
 
 ci: build
 	@echo "Success"
 
+all: build test push-dockerhub
+
 build:
-	@docker build --build-arg VERSION=$(VERSION) -t $(BUILDER_TAG) -f Dockerfile-build .
+	@docker build --build-arg VERSION=$(VERSION) -t $(BUILDER_TAG) -f files/Dockerfile-build .
 	@docker run -v m2:/root/.m2 -v `pwd`:/build $(BUILDER_TAG) cp /root/butler/butler /build
 	@docker build -t $(IMAGE_TAG) .
 
@@ -45,11 +47,11 @@ post-deploy-build:
 
 test: test-unit test-accept
 test-unit:
-	@docker build --build-arg VERSION=$(VERSION) -t $(UNIT_TESTER_TAG) -f Dockerfile-testunit .
+	@docker build --build-arg VERSION=$(VERSION) -t $(UNIT_TESTER_TAG) -f files/Dockerfile-testunit .
 	@docker run -i $(UNIT_TESTER_TAG)
 
 test-accept:
-	@docker build --build-arg VERSION=$(VERSION) -t $(ACCEPT_TESTER_TAG) -f Dockerfile-testaccept .
+	@docker build --build-arg VERSION=$(VERSION) -t $(ACCEPT_TESTER_TAG) -f files/Dockerfile-testaccept .
 	@docker run -v `pwd`/files/certs:/certs -v `pwd`/files/tests:/www --env-file <(env | egrep "(^BUT|AWS)") -i $(ACCEPT_TESTER_TAG)
 
 enter-test-unit:

VERSION

@@ -1 +1 @@
-v1.0.1
+v1.2.4

butler.go

@@ -13,26 +13,21 @@ governing permissions and limitations under the License.
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
-	"net"
-	"net/http"
 	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/adobe/butler/alog"
 	"github.com/adobe/butler/config"
-	"github.com/adobe/butler/environment"
+	"github.com/adobe/butler/internal/environment"
+	"github.com/adobe/butler/internal/monitor"
 
 	"github.com/jasonlvhit/gocron"
 	log "github.com/sirupsen/logrus"
-
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
@@ -56,70 +51,6 @@ var (
 	ButlerTesting           = false
 )
 
-// Monitor is the empty structure to be used for starting up the monitor
-// health check and prometheus metrics http endpoints.
-type Monitor struct {
-	Config *config.ButlerConfig
-}
-
-// NewMonitor returns a Monitor structure which is used to bring up the
-// monitor health check and prometheus metrics http endpoints.
-func NewMonitor(bc *config.ButlerConfig) *Monitor {
-	return &Monitor{Config: bc}
-}
-
-// MonitorOutput is the structure which holds the formatting which is output
-// to the health check monitor. When /health-check is hit, it returns this
-// structure, which is then Marshal'd to json and provided back to the end
-// user
-type MonitorOutput struct {
-	ConfigPath       string                `json:"config-path"`
-	ConfigScheme     string                `json:"config-scheme"`
-	RetrieveInterval int                   `json:"retrieve-interval"`
-	LogLevel         log.Level             `json:"log-level"`
-	ConfigSettings   config.ConfigSettings `json:"config-settings"`
-	Version          string                `json:"version"`
-}
-
-// Start turns up the http server for monitoring butler.
-func (m *Monitor) Start() {
-	mux := http.DefaultServeMux
-	mux.HandleFunc("/health-check", m.MonitorHandler)
-	mux.Handle("/metrics", promhttp.Handler())
-	loggingHandler := alog.NewApacheLoggingHandler(mux, m.Config)
-
-	server := &http.Server{
-		Handler: loggingHandler,
-	}
-
-	listener, err := net.Listen("tcp", ":8080")
-	if err != nil {
-		log.Fatalf("Error creating listener: %s", err.Error())
-	}
-	go server.Serve(listener)
-}
-
-// MonitorHandler is the handler function for the /health-check monitor
-// endpoint. It displays the JSON Marshal'd output of all the various
-// configuration options that buter gets started with, and some run time
-// information
-func (m *Monitor) MonitorHandler(w http.ResponseWriter, r *http.Request) {
-	mOut := MonitorOutput{ConfigPath: m.Config.GetPath(),
-		ConfigScheme:     m.Config.Url.Scheme,
-		RetrieveInterval: m.Config.Interval,
-		LogLevel:         m.Config.GetLogLevel(),
-		ConfigSettings:   *m.Config.Config,
-		Version:          version}
-	resp, err := json.Marshal(mOut)
-	if err != nil {
-		w.Header().Set("Content-Type", "text/html")
-		fmt.Fprintf(w, "Could not Marshal JSON, but I promise I'm up!")
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	fmt.Fprintf(w, string(resp))
-}
-
 func SetLogLevel(l string) log.Level {
 	switch strings.ToLower(l) {
 	case "debug":
@@ -272,10 +203,6 @@ func main() {
 		log.Fatalf("Cannot initialize butler config. err=%s", err.Error())
 	}
 
-	// Start up the monitor web server
-	monitor := NewMonitor(bc)
-	monitor.Start()
-
 	// Do initial grab of butler configuration file.
 	// Going to do this in an endless loop until we initially
 	// grab a configuration file.
@@ -297,6 +224,10 @@ func main() {
 		}
 	}
 
+	// Start up the monitor web server after we grab the monitor config values
+	monitor := monitor.NewMonitor(bc, monitor.MonitorOpts{Version: version})
+	monitor.Start()
+
 	sched := gocron.NewScheduler()
 	log.Debugf("main(): starting scheduler...")
 

butler_test.go

@@ -14,8 +14,10 @@ package main
 
 import (
 	. "gopkg.in/check.v1"
+
 	"testing"
 
+
 	log "github.com/sirupsen/logrus"
 )
 
@@ -49,49 +51,3 @@ func (s *ButlerTestSuite) TestSetLogLevel(c *C) {
 		c.Assert(logLevel, Equals, entry.level)
 	}
 }
-
-/*
-func (s *ButlerTestSuite) TestParseConfigFilesJsonOkDefault(c *C) {
-	err := ParseConfigFilesJson(&Files, "")
-	c.Assert(err, IsNil)
-	c.Assert(Files.Files, HasLen, 3)
-}
-
-func (s *ButlerTestSuite) TestParseConfigFilesJsonOkCustom(c *C) {
-	configFiles := `{"files": ["prometheus.yml", "alerts/commonalerts.yml", "alerts/tenant.yml", "a/b"]}`
-	err := ParseConfigFilesJson(&Files, configFiles)
-	c.Assert(err, IsNil)
-	c.Assert(Files.Files, HasLen, 4)
-}
-
-func (s *ButlerTestSuite) TestParseConfigFilesJsonNotOkCustom(c *C) {
-	configFiles := `{"files": ["prometheus.yml", "alerts/commonalerts.yml", "alerts/tenant.yml",}`
-	err := ParseConfigFilesJson(&Files, configFiles)
-	c.Assert(err, IsNil)
-	c.Assert(Files.Files, HasLen, 3)
-}
-*/
-/*
-func (s *ButlerTestSuite) TestParseConfigFilesOkDefault(c *C) {
-	err := ParseConfigFiles(&Files, FileList)
-	c.Assert(err, IsNil)
-	c.Assert(Files.Files, HasLen, 3)
-}
-
-func (s *ButlerTestSuite) TestParseConfigFilesOkCustom(c *C) {
-	configFiles := "prometheus.yml,alerts/alerts.yml  , alerts/tenant.yml, heyfoo.yml,,"
-	err := ParseConfigFiles(&Files, configFiles)
-	c.Assert(err, IsNil)
-	c.Assert(Files.Files, HasLen, 4)
-}
-
-func (s *ButlerTestSuite) TestGetPrometheusPaths(c *C) {
-	paths := GetPrometheusPaths()
-	c.Assert(paths, HasLen, 3)
-}
-
-func (s *ButlerTestSuite) TestGetPCMSUrls(c *C) {
-	urls := GetPCMSUrls()
-	c.Assert(urls, HasLen, 3)
-}
-*/

config/chan.go

@@ -18,7 +18,7 @@ import (
 	"os"
 	"sort"
 
-	"github.com/adobe/butler/stats"
+	"github.com/adobe/butler/internal/stats"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -168,7 +168,7 @@ func (c *ConfigChanEvent) CopyPrimaryConfigFiles(opts map[string]*ManagerOpts) b
 
 	out, err := os.OpenFile(c.TmpFile.Name(), os.O_WRONLY|os.O_TRUNC, 0644)
 	if err != nil {
-		log.Infof("ConfigChanEvent::CopyPrimaryConfigFiles(): Could not process and merge new %s err=%s.", c.ConfigFile, err.Error())
+		log.Infof("ConfigChanEvent::CopyPrimaryConfigFiles(): Could not process and merge new %v err=%s.", c.ConfigFile, err.Error())
 		stats.SetButlerConfigVal(stats.FAILURE, "local", stats.GetStatsLabel(*c.ConfigFile))
 		c.CleanTmpFiles()
 		return false
@@ -179,15 +179,15 @@ func (c *ConfigChanEvent) CopyPrimaryConfigFiles(opts map[string]*ManagerOpts) b
 				if t.Name == f {
 					in, err := os.Open(t.File)
 					if err != nil {
-						log.Infof("ConfigChanEvent::CopyPrimaryConfigFiles(): Could not process and merge new %s err=%s.", c.ConfigFile, err.Error())
+						log.Infof("ConfigChanEvent::CopyPrimaryConfigFiles(): Could not process and merge new %v err=%s.", c.ConfigFile, err.Error())
 						stats.SetButlerConfigVal(stats.FAILURE, "local", stats.GetStatsLabel(t.Name))
 						c.CleanTmpFiles()
 						out.Close()
 						return false
 					}
 					_, err = io.Copy(out, in)
 					if err != nil {
-						log.Infof("ConfigChanEvent::CopyPrimaryConfigFiles(): Could not process and merge new %s err=%s.", c.ConfigFile, err.Error())
+						log.Infof("ConfigChanEvent::CopyPrimaryConfigFiles(): Could not process and merge new %v err=%s.", c.ConfigFile, err.Error())
 						stats.SetButlerConfigVal(stats.FAILURE, "local", stats.GetStatsLabel(t.Name))
 						c.CleanTmpFiles()
 						out.Close()

config/config.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/adobe/butler/config/methods"
-	"github.com/adobe/butler/environment"
+	"github.com/adobe/butler/internal/environment"
 
 	"github.com/hashicorp/go-retryablehttp"
 	log "github.com/sirupsen/logrus"
@@ -155,8 +155,31 @@ func (c *ConfigSettings) ParseConfig(config []byte) error {
 		// enable http logging
 	}
 
-	log.Debugf("ConfigSettings::ParseConfig(): globals.config-managers=%#v", Config.Globals.Managers)
-	log.Debugf("ConfigSettings::ParseConfig(): len(globals.config-managers)=%v", len(Config.Globals.Managers))
+	// Let's determine the http proto and the port
+	envHttpPort, _ := strconv.Atoi(environment.GetVar(Config.Globals.CfgHttpPort))
+	if envHttpPort == 0 {
+		Config.Globals.HttpPort = 8080
+	} else {
+		Config.Globals.HttpPort = envHttpPort
+	}
+
+	Config.Globals.HttpProto = strings.ToLower(environment.GetVar(Config.Globals.CfgHttpProto))
+	if (Config.Globals.HttpProto != "http") && (Config.Globals.HttpProto != "https") {
+		Config.Globals.HttpProto = "http"
+	}
+
+	if Config.Globals.HttpProto == "https" {
+		Config.Globals.HttpTlsCert = environment.GetVar(Config.Globals.CfgHttpTlsCert)
+		Config.Globals.HttpTlsKey = environment.GetVar(Config.Globals.CfgHttpTlsKey)
+		if (Config.Globals.HttpTlsCert == "") || (Config.Globals.HttpTlsKey == "") {
+			if Config.Globals.ExitOnFailure {
+				log.Fatalf("ConfigSetings::ParseConfig(): globlals.http-proto set to \"https\" but no cert and/or key defined! exiting...")
+			} else {
+				log.Debugf("ConfigSetings::ParseConfig(): globlals.http-proto set to \"https\" but no cert and/or key defined")
+				return errors.New("globals.http-proto set to https but no tls options defined.")
+			}
+		}
+	}
 
 	// If there are no entries for config-managers, then the Unmarshal will create an empty array
 	if len(Config.Globals.Managers) < 1 {

config/handler.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/adobe/butler/config/methods"
 	"github.com/adobe/butler/config/reloaders"
-	"github.com/adobe/butler/stats"
+	"github.com/adobe/butler/internal/stats"
 
 	"github.com/jasonlvhit/gocron"
 	log "github.com/sirupsen/logrus"

config/helpers.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/adobe/butler/config/methods"
 	"github.com/adobe/butler/config/reloaders"
-	"github.com/adobe/butler/environment"
-	"github.com/adobe/butler/stats"
+	"github.com/adobe/butler/internal/environment"
+	"github.com/adobe/butler/internal/stats"
 
 	"github.com/Jeffail/gabs"
 	"github.com/hashicorp/go-retryablehttp"
@@ -181,7 +181,6 @@ func removeButlerHeaderFooter(file interface{}) error {
 	default:
 		return nil
 	}
-	return nil
 }
 
 func runTextValidate(f *bytes.Reader, m string) error {

config/manager.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/adobe/butler/config/methods"
 	"github.com/adobe/butler/config/reloaders"
-	"github.com/adobe/butler/stats"
+	"github.com/adobe/butler/internal/stats"
 
 	"strings"
 
@@ -63,7 +63,7 @@ type ManagerOpts struct {
 	AdditionalConfigsFullLocalPaths []string       `json:"-"`
 	ContentType                     string         `mapstructure:"content-type" json:"content-type"`
 	Opts                            methods.Method `json:"opts"`
-	parentManager                   string         `json:"-"`
+	parentManager                   string
 }
 
 func (bm *Manager) Reload() error {
@@ -90,7 +90,7 @@ func (bm *Manager) DownloadPrimaryConfigFiles(c chan ChanEvent) error {
 	// Create a temporary file for the merged prometheus configurations.
 	tmpFile, err := ioutil.TempFile("/tmp", "bcmsfile")
 	if err != nil {
-		msg := fmt.Sprintf("Manager::DownloadPrimaryConfigFiles(): Could not create temporary file . err=v", err.Error())
+		msg := fmt.Sprintf("Manager::DownloadPrimaryConfigFiles(): Could not create temporary file . err=%s", err.Error())
 		log.Fatal(msg)
 	}
 	Chan.TmpFile = tmpFile

config/methods/blob.go

@@ -19,7 +19,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/adobe/butler/environment"
+	"github.com/adobe/butler/internal/environment"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
 	//log "github.com/sirupsen/logrus"

config/methods/etcd.go

@@ -21,7 +21,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/adobe/butler/environment"
+	"github.com/adobe/butler/internal/environment"
 
 	"github.com/coreos/etcd/client"
 	log "github.com/sirupsen/logrus"

config/methods/file.go

@@ -19,7 +19,7 @@ import (
 	"io/ioutil"
 	"net/url"
 
-	"github.com/adobe/butler/environment"
+	"github.com/adobe/butler/internal/environment"
 
 	//log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"