[FAB-2156] Move orderer viper utils to common

https://jira.hyperledger.org/browse/FAB-2156

Before moving the orderer genesis tooling to common, it requires some
viper extensions which resided in the orderer, this CR moves them to
common.

Change-Id: I5c68cad8903819d490a0e908a7ac07a2f6d86fdc
Signed-off-by: Jason Yellick <jyellick@us.ibm.com>

common/viperutil/config_test.go

@@ -0,0 +1,312 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package viperutil
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/hyperledger/fabric/orderer/mocks/util"
+	"github.com/spf13/viper"
+)
+
+const Prefix = "VIPERUTIL"
+
+type testSlice struct {
+	Inner struct {
+		Slice []string
+	}
+}
+
+func TestEnvSlice(t *testing.T) {
+	envVar := "VIPERUTIL_INNER_SLICE"
+	envVal := "[a, b, c]"
+	os.Setenv(envVar, envVal)
+	defer os.Unsetenv(envVar)
+	config := viper.New()
+	config.SetEnvPrefix(Prefix)
+	config.AutomaticEnv()
+	replacer := strings.NewReplacer(".", "_")
+	config.SetEnvKeyReplacer(replacer)
+	config.SetConfigType("yaml")
+
+	data := "---\nInner:\n    Slice: [d,e,f]"
+
+	err := config.ReadConfig(bytes.NewReader([]byte(data)))
+
+	if err != nil {
+		t.Fatalf("Error reading %s plugin config: %s", Prefix, err)
+	}
+
+	var uconf testSlice
+
+	err = EnhancedExactUnmarshal(config, &uconf)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal with: %s", err)
+	}
+
+	expected := []string{"a", "b", "c"}
+	if !reflect.DeepEqual(uconf.Inner.Slice, expected) {
+		t.Fatalf("Did not get back the right slice, expeced: %v got %v", expected, uconf.Inner.Slice)
+	}
+}
+
+type testByteSize struct {
+	Inner struct {
+		ByteSize uint32
+	}
+}
+
+func TestByteSize(t *testing.T) {
+	config := viper.New()
+	config.SetConfigType("yaml")
+
+	testCases := []struct {
+		data     string
+		expected uint32
+	}{
+		{"", 0},
+		{"42", 42},
+		{"42k", 42 * 1024},
+		{"42kb", 42 * 1024},
+		{"42K", 42 * 1024},
+		{"42KB", 42 * 1024},
+		{"42 K", 42 * 1024},
+		{"42 KB", 42 * 1024},
+		{"42m", 42 * 1024 * 1024},
+		{"42mb", 42 * 1024 * 1024},
+		{"42M", 42 * 1024 * 1024},
+		{"42MB", 42 * 1024 * 1024},
+		{"42 M", 42 * 1024 * 1024},
+		{"42 MB", 42 * 1024 * 1024},
+		{"3g", 3 * 1024 * 1024 * 1024},
+		{"3gb", 3 * 1024 * 1024 * 1024},
+		{"3G", 3 * 1024 * 1024 * 1024},
+		{"3GB", 3 * 1024 * 1024 * 1024},
+		{"3 G", 3 * 1024 * 1024 * 1024},
+		{"3 GB", 3 * 1024 * 1024 * 1024},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.data, func(t *testing.T) {
+			data := fmt.Sprintf("---\nInner:\n    ByteSize: %s", tc.data)
+			err := config.ReadConfig(bytes.NewReader([]byte(data)))
+			if err != nil {
+				t.Fatalf("Error reading config: %s", err)
+			}
+			var uconf testByteSize
+			err = EnhancedExactUnmarshal(config, &uconf)
+			if err != nil {
+				t.Fatalf("Failed to unmarshal with: %s", err)
+			}
+			if uconf.Inner.ByteSize != tc.expected {
+				t.Fatalf("Did not get back the right byte size, expeced: %v got %v", tc.expected, uconf.Inner.ByteSize)
+			}
+		})
+	}
+}
+
+func TestByteSizeOverflow(t *testing.T) {
+	config := viper.New()
+	config.SetConfigType("yaml")
+
+	data := "---\nInner:\n    ByteSize: 4GB"
+	err := config.ReadConfig(bytes.NewReader([]byte(data)))
+	if err != nil {
+		t.Fatalf("Error reading config: %s", err)
+	}
+	var uconf testByteSize
+	err = EnhancedExactUnmarshal(config, &uconf)
+	if err == nil {
+		t.Fatalf("Should have failed to unmarshal")
+	}
+}
+
+type stringFromFileConfig struct {
+	Inner struct {
+		Single   string
+		Multiple []string
+	}
+}
+
+func TestStringNotFromFile(t *testing.T) {
+
+	expectedValue := "expected_value"
+	yaml := fmt.Sprintf("---\nInner:\n  Single: %s\n", expectedValue)
+
+	config := viper.New()
+	config.SetConfigType("yaml")
+
+	if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil {
+		t.Fatalf("Error reading config: %s", err)
+	}
+
+	var uconf stringFromFileConfig
+	if err := EnhancedExactUnmarshal(config, &uconf); err != nil {
+		t.Fatalf("Failed to unmarshall: %s", err)
+	}
+
+	if uconf.Inner.Single != expectedValue {
+		t.Fatalf(`Expected: "%s", Actual: "%s"`, expectedValue, uconf.Inner.Single)
+	}
+
+}
+
+func TestStringFromFile(t *testing.T) {
+
+	expectedValue := "this is the text in the file"
+
+	// create temp file
+	file, err := ioutil.TempFile(os.TempDir(), "test")
+	if err != nil {
+		t.Fatalf("Unable to create temp file.")
+	}
+	defer os.Remove(file.Name())
+
+	// write temp file
+	if err = ioutil.WriteFile(file.Name(), []byte(expectedValue), 0777); err != nil {
+		t.Fatalf("Unable to write to temp file.")
+	}
+
+	yaml := fmt.Sprintf("---\nInner:\n  Single:\n    File: %s", file.Name())
+
+	config := viper.New()
+	config.SetConfigType("yaml")
+
+	if err = config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil {
+		t.Fatalf("Error reading config: %s", err)
+	}
+	var uconf stringFromFileConfig
+	if err = EnhancedExactUnmarshal(config, &uconf); err != nil {
+		t.Fatalf("Failed to unmarshall: %s", err)
+	}
+
+	if uconf.Inner.Single != expectedValue {
+		t.Fatalf(`Expected: "%s", Actual: "%s"`, expectedValue, uconf.Inner.Single)
+	}
+}
+
+func TestPEMBlocksFromFile(t *testing.T) {
+
+	// create temp file
+	file, err := ioutil.TempFile(os.TempDir(), "test")
+	if err != nil {
+		t.Fatalf("Unable to create temp file.")
+	}
+	defer os.Remove(file.Name())
+
+	numberOfCertificates := 3
+	var pems []byte
+	for i := 0; i < numberOfCertificates; i++ {
+		publicKeyCert, _, err := util.GenerateMockPublicPrivateKeyPairPEM(true)
+		if err != nil {
+			t.Fatalf("Enable to generate a signer certificate: %v", err)
+		}
+		pems = append(pems, publicKeyCert...)
+	}
+
+	// write temp file
+	if err := ioutil.WriteFile(file.Name(), pems, 0666); err != nil {
+		t.Fatalf("Unable to write to temp file: %v", err)
+	}
+
+	yaml := fmt.Sprintf("---\nInner:\n  Multiple:\n    File: %s", file.Name())
+
+	config := viper.New()
+	config.SetConfigType("yaml")
+
+	if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil {
+		t.Fatalf("Error reading config: %v", err)
+	}
+	var uconf stringFromFileConfig
+	if err := EnhancedExactUnmarshal(config, &uconf); err != nil {
+		t.Fatalf("Failed to unmarshall: %v", err)
+	}
+
+	if len(uconf.Inner.Multiple) != 3 {
+		t.Fatalf(`Expected: "%v", Actual: "%v"`, numberOfCertificates, len(uconf.Inner.Multiple))
+	}
+}
+
+func TestStringFromFileNotSpecified(t *testing.T) {
+
+	yaml := fmt.Sprintf("---\nInner:\n  Single:\n    File:\n")
+
+	config := viper.New()
+	config.SetConfigType("yaml")
+
+	if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil {
+		t.Fatalf("Error reading config: %s", err)
+	}
+	var uconf stringFromFileConfig
+	if err := EnhancedExactUnmarshal(config, &uconf); err == nil {
+		t.Fatalf("Should of failed to unmarshall.")
+	}
+
+}
+
+func TestStringFromFileEnv(t *testing.T) {
+
+	expectedValue := "this is the text in the file"
+
+	// create temp file
+	file, err := ioutil.TempFile(os.TempDir(), "test")
+	if err != nil {
+		t.Fatalf("Unable to create temp file.")
+	}
+	defer os.Remove(file.Name())
+
+	// write temp file
+	if err = ioutil.WriteFile(file.Name(), []byte(expectedValue), 0777); err != nil {
+		t.Fatalf("Unable to write to temp file.")
+	}
+
+	envVar := "VIPERUTIL_INNER_SINGLE_FILE"
+	envVal := file.Name()
+	os.Setenv(envVar, envVal)
+	defer os.Unsetenv(envVar)
+	config := viper.New()
+	config.SetEnvPrefix(Prefix)
+	config.AutomaticEnv()
+	replacer := strings.NewReplacer(".", "_")
+	config.SetEnvKeyReplacer(replacer)
+	config.SetConfigType("yaml")
+
+	data := "---\nInner:\n  Single:\n    File: wrong_file"
+
+	if err = config.ReadConfig(bytes.NewReader([]byte(data))); err != nil {
+		t.Fatalf("Error reading %s plugin config: %s", Prefix, err)
+	}
+
+	var uconf stringFromFileConfig
+
+	err = EnhancedExactUnmarshal(config, &uconf)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal with: %s", err)
+	}
+
+	t.Log(uconf.Inner.Single)
+
+	if !reflect.DeepEqual(uconf.Inner.Single, expectedValue) {
+		t.Fatalf(`Expected: "%v",  Actual: "%v"`, expectedValue, uconf.Inner.Single)
+	}
+}

orderer/localconfig/config_util.go → common/viperutil/config_util.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package config
+package viperutil
 
 import (
 	"fmt"
@@ -30,9 +30,12 @@ import (
 	"encoding/pem"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/op/go-logging"
 	"github.com/spf13/viper"
 )
 
+var logger = logging.MustGetLogger("viperutil")
+
 func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} {
 	result := make(map[string]interface{})
 	for key := range nodeKeys {
@@ -228,10 +231,10 @@ func pemBlocksFromFileDecodeHook() mapstructure.DecodeHookFunc {
 	}
 }
 
-// ExactWithDateUnmarshal is intended to unmarshal a config file into a structure
+// EnhancedExactUnmarshal is intended to unmarshal a config file into a structure
 // producing error when extraneous variables are introduced and supporting
 // the time.Duration type
-func ExactWithDateUnmarshal(v *viper.Viper, output interface{}) error {
+func EnhancedExactUnmarshal(v *viper.Viper, output interface{}) error {
 	baseKeys := v.AllSettings() // AllKeys doesn't actually return all keys, it only returns the base ones
 	leafKeys := getKeysRecursively("", v, baseKeys)
 

orderer/localconfig/config.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hyperledger/fabric/common/viperutil"
+
 	"github.com/Shopify/sarama"
 	"github.com/op/go-logging"
 	"github.com/spf13/viper"
@@ -358,7 +360,7 @@ func LoadGenesis() *GenesisTopLevel {
 
 	var uconf GenesisTopLevel
 
-	err = ExactWithDateUnmarshal(config, &uconf)
+	err = viperutil.EnhancedExactUnmarshal(config, &uconf)
 	if err != nil {
 		panic(fmt.Errorf("Error unmarshaling into structure: %s", err))
 	}
@@ -407,7 +409,7 @@ func Load() *TopLevel {
 
 	var uconf TopLevel
 
-	err = ExactWithDateUnmarshal(config, &uconf)
+	err = viperutil.EnhancedExactUnmarshal(config, &uconf)
 	if err != nil {
 		panic(fmt.Errorf("Error unmarshaling into structure: %s", err))
 	}