Added support for using AWS encrypted BLS key files (#2650)

* added bls support, cleaned the code

* fixed code per review

cmd/harmony/main.go

@@ -107,6 +107,7 @@ var (
 	shardID            = flag.Int("shard_id", -1, "the shard ID of this node")
 	enableMemProfiling = flag.Bool("enableMemProfiling", false, "Enable memsize logging.")
 	enableGC           = flag.Bool("enableGC", true, "Enable calling garbage collector manually .")
+	cmkEncryptedBLSKey = flag.String("aws_blskey", "", "The aws CMK encrypted bls private key file.")
 	blsKeyFile         = flag.String("blskey_file", "", "The encrypted file of bls serialized private key by passphrase.")
 	blsFolder          = flag.String("blsfolder", ".hmy/blskeys", "The folder that stores the bls keys and corresponding passphrases; e.g. <blskey>.key and <blskey>.pass; all bls keys mapped to same shard")
 	blsPass            = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.")
@@ -144,6 +145,8 @@ var (
 	webHookYamlPath = flag.String(
 		"webhook_yaml", "", "path for yaml config reporting double signing",
 	)
+	// aws credentials
+	awsSettingString = ""
 )
 
 func initSetup() {
@@ -154,7 +157,9 @@ func initSetup() {
 	}
 
 	// maybe request passphrase for bls key.
-	passphraseForBLS()
+	if *cmkEncryptedBLSKey == "" {
+		passphraseForBLS()
+	}
 
 	// Configure log parameters
 	utils.SetLogContext(*port, *ip)
@@ -278,6 +283,8 @@ func setupStakingNodeAccount() error {
 func readMultiBLSKeys(consensusMultiBLSPriKey *multibls.PrivateKey, consensusMultiBLSPubKey *multibls.PublicKey) error {
 	keyPasses := map[string]string{}
 	blsKeyFiles := []os.FileInfo{}
+	awsEncryptedBLSKeyFiles := []os.FileInfo{}
+
 	if err := filepath.Walk(*blsFolder, func(path string, info os.FileInfo, err error) error {
 		if info.IsDir() {
 			return nil
@@ -294,9 +301,11 @@ func readMultiBLSKeys(consensusMultiBLSPriKey *multibls.PrivateKey, consensusMul
 			}
 			name := fullName[:len(fullName)-len(ext)]
 			keyPasses[name] = passphrase
+		} else if ext == ".bls" {
+			awsEncryptedBLSKeyFiles = append(awsEncryptedBLSKeyFiles, info)
 		} else {
 			return errors.Errorf(
-				"[Multi-BLS] found file: %s that does not have .key or .pass file extension",
+				"[Multi-BLS] found file: %s that does not have .bls, .key or .pass file extension",
 				path,
 			)
 		}
@@ -309,23 +318,41 @@ func readMultiBLSKeys(consensusMultiBLSPriKey *multibls.PrivateKey, consensusMul
 		)
 		os.Exit(100)
 	}
-	if len(blsKeyFiles) > *maxBLSKeysPerNode {
+
+	keyFiles := []os.FileInfo{}
+	legacyBLSFile := true
+
+	if len(awsEncryptedBLSKeyFiles) > 0 {
+		keyFiles = awsEncryptedBLSKeyFiles
+		legacyBLSFile = false
+	} else {
+		keyFiles = blsKeyFiles
+	}
+
+	if len(keyFiles) > *maxBLSKeysPerNode {
 		fmt.Fprintf(os.Stderr,
 			"[Multi-BLS] maximum number of bls keys per node is %d, found: %d\n",
 			*maxBLSKeysPerNode,
-			len(blsKeyFiles),
+			len(keyFiles),
 		)
 		os.Exit(100)
 	}
-	for _, blsKeyFile := range blsKeyFiles {
-		fullName := blsKeyFile.Name()
-		ext := filepath.Ext(fullName)
-		name := fullName[:len(fullName)-len(ext)]
-		if val, ok := keyPasses[name]; ok {
-			blsPassphrase = val
-		}
+
+	for _, blsKeyFile := range keyFiles {
+		var consensusPriKey *bls.SecretKey
+		var err error
 		blsKeyFilePath := path.Join(*blsFolder, blsKeyFile.Name())
-		consensusPriKey, err := blsgen.LoadBLSKeyWithPassPhrase(blsKeyFilePath, blsPassphrase)
+		if legacyBLSFile {
+			fullName := blsKeyFile.Name()
+			ext := filepath.Ext(fullName)
+			name := fullName[:len(fullName)-len(ext)]
+			if val, ok := keyPasses[name]; ok {
+				blsPassphrase = val
+			}
+			consensusPriKey, err = blsgen.LoadBLSKeyWithPassPhrase(blsKeyFilePath, blsPassphrase)
+		} else {
+			consensusPriKey, err = blsgen.LoadAwsCMKEncryptedBLSKey(blsKeyFilePath, awsSettingString)
+		}
 		if err != nil {
 			return err
 		}
@@ -347,6 +374,15 @@ func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) multibls.PublicKey {
 			fmt.Fprintf(os.Stderr, "ERROR when loading bls key, err :%v\n", err)
 			os.Exit(100)
 		}
+		multibls.AppendPriKey(consensusMultiPriKey, consensusPriKey)
+		multibls.AppendPubKey(consensusMultiPubKey, consensusPriKey.GetPublicKey())
+	} else if *cmkEncryptedBLSKey != "" {
+		consensusPriKey, err := blsgen.LoadAwsCMKEncryptedBLSKey(*cmkEncryptedBLSKey, awsSettingString)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "ERROR when loading aws CMK encrypted bls key, err :%v\n", err)
+			os.Exit(100)
+		}
+
 		multibls.AppendPriKey(consensusMultiPriKey, consensusPriKey)
 		multibls.AppendPubKey(consensusMultiPubKey, consensusPriKey.GetPublicKey())
 	} else {
@@ -654,6 +690,9 @@ func main() {
 	// build time.
 	os.Setenv("GODEBUG", "netdns=go")
 
+	// Get aws credentials from prompt timeout 1 second if there's no input
+	awsSettingString, _ = blsgen.Readln(1 * time.Second)
+
 	flag.Var(&p2putils.BootNodes, "bootnodes", "a list of bootnode multiaddress (delimited by ,)")
 	flag.Parse()
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Workiva/go-datastructures v1.0.50
 	github.com/allegro/bigcache v1.2.1 // indirect
 	github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08 // indirect
+	github.com/aws/aws-sdk-go v1.30.1
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/cespare/cp v1.1.1
@@ -45,7 +46,7 @@ require (
 	github.com/multiformats/go-multiaddr-net v0.1.2
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/pborman/uuid v1.2.0
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v0.9.3
 	github.com/prometheus/common v0.4.1 // indirect
 	github.com/prometheus/procfs v0.0.3 // indirect
@@ -55,7 +56,7 @@ require (
 	github.com/shirou/gopsutil v2.18.12+incompatible
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.6.1
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.1
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
 	github.com/uber/jaeger-client-go v2.20.1+incompatible // indirect
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect

internal/blsgen/lib.go

@@ -1,22 +1,34 @@
 package blsgen
 
 import (
+	"bufio"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/md5"
 	"crypto/rand"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/kms"
 	ffi_bls "github.com/harmony-one/bls/ffi/go/bls"
-
 	"github.com/harmony-one/harmony/crypto/bls"
+	"github.com/pkg/errors"
 )
 
+type awsConfiguration struct {
+	AccessKey string `json:"aws-access-key-id"`
+	SecretKey string `json:"aws-secret-access-key"`
+	Region    string `json:"aws-region"`
+}
+
 func toISO8601(t time.Time) string {
 	var tz string
 	name, offset := t.Zone()
@@ -94,6 +106,86 @@ func LoadBLSKeyWithPassPhrase(fileName, passphrase string) (*ffi_bls.SecretKey,
 	return priKey, nil
 }
 
+// Readln reads aws configuratoin from prompt with a timeout
+func Readln(timeout time.Duration) (string, error) {
+	s := make(chan string)
+	e := make(chan error)
+
+	go func() {
+		reader := bufio.NewReader(os.Stdin)
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			e <- err
+		} else {
+			s <- line
+		}
+		close(s)
+		close(e)
+	}()
+
+	select {
+	case line := <-s:
+		return line, nil
+	case err := <-e:
+		return "", err
+	case <-time.After(timeout):
+		return "", errors.New("Timeout")
+	}
+}
+
+// LoadAwsCMKEncryptedBLSKey loads aws encrypted bls key.
+func LoadAwsCMKEncryptedBLSKey(fileName, awsSettingString string) (*ffi_bls.SecretKey, error) {
+	if awsSettingString == "" {
+		return nil, errors.New("aws credential is not set")
+	}
+
+	var awsConfig awsConfiguration
+	if err := json.Unmarshal([]byte(awsSettingString), &awsConfig); err != nil {
+		return nil, errors.New(awsSettingString + " is not a valid JSON string for setting aws configuration.")
+	}
+
+	// Initialize a session that the aws SDK uses to load
+	sess, err := session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	})
+
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create aws session")
+	}
+
+	// Create KMS service client
+	svc := kms.New(sess, &aws.Config{
+		//Region: aws.String("us-east-1"),
+		Region:      aws.String(awsConfig.Region),
+		Credentials: credentials.NewStaticCredentials(awsConfig.AccessKey, awsConfig.SecretKey, ""),
+	})
+
+	encryptedPrivateKeyBytes, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "fail read at: %s", fileName)
+	}
+
+	unhexed := make([]byte, hex.DecodedLen(len(encryptedPrivateKeyBytes)))
+	if _, err = hex.Decode(unhexed, encryptedPrivateKeyBytes); err != nil {
+		return nil, err
+	}
+
+	clearKey, err := svc.Decrypt(&kms.DecryptInput{
+		CiphertextBlob: unhexed,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	priKey := &ffi_bls.SecretKey{}
+	if err = priKey.DeserializeHexStr(hex.EncodeToString(clearKey.Plaintext)); err != nil {
+		return nil, errors.Wrapf(err, "failed to deserialize the decrypted bls private key")
+	}
+
+	return priKey, nil
+}
+
 func createHash(key string) string {
 	hasher := md5.New()
 	hasher.Write([]byte(key))