Merge "Add orderer config mechanism" into feature/convergence

bddtests/docker-compose-orderer-solo.yml

@@ -1,12 +1,14 @@
 orderer0:
   image: hyperledger/fabric-orderer
   environment:
-    - ORDERER_LISTEN_ADDRESS=0.0.0.0
-    - ORDERER_LISTEN_PORT=5005
-    #- ORDERER_WINDOW_SIZE_MAX=1000 # TODO (implement)
-    #- ORDERER_BATCH_TIMEOUT=10s # TODO (implement)
-    #- ORDERER_BATCH_SIZE=10 # TODO (implement)
-    #- ORDERER_BLOCK_HISTORY_SIZE=100 # TODO (implement)
+    - ORDERER_GENERAL_ORDERERTYPE=solo
+    - ORDERER_GENERAL_LEDGERTYPE=ram
+    - ORDERER_GENERAL_BATCHTIMEOUT=10s
+    - ORDERER_GENERAL_BATCHSIZE=10
+    - ORDERER_GENERAL_MAXWINDOWSIZE=1000
+    - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
+    - ORDERER_GENERAL_LISTENPORT=5005
+    - ORDERER_RAMLEDGER_HISTORY_SIZE=100
 
   working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderer
   command: orderer

orderer/config/config.go

@@ -0,0 +1,167 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/op/go-logging"
+	"github.com/spf13/viper"
+)
+
+var logger = logging.MustGetLogger("orderer/config")
+
+func init() {
+	logging.SetLevel(logging.DEBUG, "")
+}
+
+// Prefix is the default config prefix for the orderer
+const Prefix string = "ORDERER"
+
+// General contains config which should be common among all orderer types
+type General struct {
+	OrdererType   string
+	LedgerType    string
+	BatchTimeout  time.Duration
+	BatchSize     uint
+	QueueSize     uint
+	MaxWindowSize uint
+	ListenAddress string
+	ListenPort    uint16
+}
+
+// RAMLedger contains config for the RAM ledger
+type RAMLedger struct {
+	HistorySize uint
+}
+
+// FileLedger contains config for the File ledger
+type FileLedger struct {
+	Location string
+	Prefix   string
+}
+
+// TopLevel directly corresponds to the orderer config yaml
+// Note, for non 1-1 mappings, you may append
+// something like `mapstructure:"weirdFoRMat"` to
+// modify the default mapping, see the "Unmarshal"
+// section of https://github.com/spf13/viper for more info
+type TopLevel struct {
+	General    General
+	RAMLedger  RAMLedger
+	FileLedger FileLedger
+}
+
+var defaults = TopLevel{
+	General: General{
+		OrdererType:   "solo",
+		LedgerType:    "ram",
+		BatchTimeout:  10 * time.Second,
+		BatchSize:     10,
+		QueueSize:     1000,
+		MaxWindowSize: 1000,
+		ListenAddress: "127.0.0.1",
+		ListenPort:    5151,
+	},
+	RAMLedger: RAMLedger{
+		HistorySize: 10000,
+	},
+	FileLedger: FileLedger{
+		Location: "",
+		Prefix:   "hyperledger-fabric-rawledger",
+	},
+}
+
+func (c *TopLevel) completeInitialization() {
+	defer logger.Infof("Validated configuration to: %+v", c)
+
+	for {
+		switch {
+		case c.General.OrdererType == "":
+			logger.Infof("General.OrdererType unset, setting to %s", defaults.General.OrdererType)
+			c.General.OrdererType = defaults.General.OrdererType
+		case c.General.LedgerType == "":
+			logger.Infof("General.LedgerType unset, setting to %s", defaults.General.LedgerType)
+			c.General.LedgerType = defaults.General.LedgerType
+		case c.General.BatchTimeout == 0:
+			logger.Infof("General.BatchTimeout unset, setting to %s", defaults.General.BatchTimeout)
+			c.General.BatchTimeout = defaults.General.BatchTimeout
+		case c.General.BatchSize == 0:
+			logger.Infof("General.BatchSize unset, setting to %s", defaults.General.BatchSize)
+			c.General.BatchSize = defaults.General.BatchSize
+		case c.General.QueueSize == 0:
+			logger.Infof("General.QueueSize unset, setting to %s", defaults.General.QueueSize)
+			c.General.QueueSize = defaults.General.QueueSize
+		case c.General.MaxWindowSize == 0:
+			logger.Infof("General.MaxWindowSize unset, setting to %s", defaults.General.MaxWindowSize)
+			c.General.MaxWindowSize = defaults.General.MaxWindowSize
+		case c.General.ListenAddress == "":
+			logger.Infof("General.ListenAddress unset, setting to %s", defaults.General.ListenAddress)
+			c.General.ListenAddress = defaults.General.ListenAddress
+		case c.General.ListenPort == 0:
+			logger.Infof("General.ListenPort unset, setting to %s", defaults.General.ListenPort)
+			c.General.ListenPort = defaults.General.ListenPort
+		case c.FileLedger.Prefix == "":
+			logger.Infof("FileLedger.Prefix unset, setting to %s", defaults.FileLedger.Prefix)
+			c.FileLedger.Prefix = defaults.FileLedger.Prefix
+		default:
+			return
+		}
+	}
+}
+
+// Load parses the orderer.yaml file and environment, producing a struct suitable for config use
+func Load() *TopLevel {
+	config := viper.New()
+
+	// for environment variables
+	config.SetEnvPrefix(Prefix)
+	config.AutomaticEnv()
+	replacer := strings.NewReplacer(".", "_")
+	config.SetEnvKeyReplacer(replacer)
+
+	config.SetConfigName("orderer")
+	config.AddConfigPath("./")
+	config.AddConfigPath("../orderer/")
+	config.AddConfigPath("../../orderer/")
+	// Path to look for the config file in based on GOPATH
+	gopath := os.Getenv("GOPATH")
+	for _, p := range filepath.SplitList(gopath) {
+		ordererPath := filepath.Join(p, "src/github.com/hyperledger/fabric/orderer/")
+		config.AddConfigPath(ordererPath)
+	}
+
+	err := config.ReadInConfig()
+	if err != nil {
+		panic(fmt.Errorf("Error reading %s plugin config: %s", Prefix, err))
+	}
+
+	var uconf TopLevel
+
+	err = ExactWithDateUnmarshal(config, &uconf)
+	if err != nil {
+		panic(fmt.Errorf("Error unmarshaling into structure: %s", err))
+	}
+
+	uconf.completeInitialization()
+
+	return &uconf
+}

orderer/config/config_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/spf13/viper"
+)
+
+func TestGoodConfig(t *testing.T) {
+	config := Load()
+	if config == nil {
+		t.Fatalf("Could not load config")
+	}
+	t.Logf("%+v", config)
+}
+
+func TestBadConfig(t *testing.T) {
+	config := viper.New()
+	config.SetConfigName("orderer")
+	config.AddConfigPath("../")
+
+	err := config.ReadInConfig()
+	if err != nil {
+		t.Fatalf("Error reading %s plugin config: %s", Prefix, err)
+	}
+
+	var uconf struct{}
+
+	err = ExactWithDateUnmarshal(config, &uconf)
+	if err == nil {
+		t.Fatalf("Should have failed to unmarshal")
+	}
+}
+
+// TestEnvInnerVar verifies that with the Unmarshal function that
+// the environmental overrides still work on internal vars.  This was
+// a bug in the original viper implementation that is worked around in
+// the Load codepath for now
+func TestEnvInnerVar(t *testing.T) {
+	envVar := "ORDERER_GENERAL_LISTENPORT"
+	envVal := uint16(80)
+	os.Setenv(envVar, fmt.Sprintf("%d", envVal))
+	defer os.Unsetenv(envVar)
+	config := Load()
+
+	if config == nil {
+		t.Fatalf("Could not load config")
+	}
+
+	if config.General.ListenPort != envVal {
+		t.Fatalf("Environmental override of inner config did not work")
+	}
+}

orderer/config/config_util.go

@@ -0,0 +1,69 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+)
+
+func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	for key := range nodeKeys {
+		fqKey := base + key
+		val := v.Get(fqKey)
+		if m, ok := val.(map[interface{}]interface{}); ok {
+			logger.Debugf("Found map value for %s", fqKey)
+			tmp := make(map[string]interface{})
+			for ik, iv := range m {
+				cik, ok := ik.(string)
+				if !ok {
+					panic("Non string key-entry")
+				}
+				tmp[cik] = iv
+			}
+			result[key] = getKeysRecursively(fqKey+".", v, tmp)
+		} else {
+			logger.Debugf("Found real value for %s setting to %T %v", fqKey, val, val)
+			result[key] = val
+		}
+	}
+	return result
+}
+
+// ExactWithDateUnmarshal is intended to unmarshal a config file into a structure
+// producing error when extraneous variables are introduced and supporting
+// the time.Duration type
+func ExactWithDateUnmarshal(v *viper.Viper, output interface{}) error {
+	baseKeys := v.AllSettings() // AllKeys doesn't actually return all keys, it only returns the base ones
+	leafKeys := getKeysRecursively("", v, baseKeys)
+
+	logger.Infof("%+v", leafKeys)
+	config := &mapstructure.DecoderConfig{
+		ErrorUnused:      true,
+		Metadata:         nil,
+		Result:           output,
+		WeaklyTypedInput: true,
+		DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
+	}
+
+	decoder, err := mapstructure.NewDecoder(config)
+	if err != nil {
+		return err
+	}
+	return decoder.Decode(leafKeys)
+}