Add AcraTranslator service T663 (#213)

* acra reader: entry point

* deadline listener; refactor acra-server

* base part of connection handling (#197)

* base part of connection handling

* save connections by descriptor

* add a little bit graceful stop service

* basic http /decrypt handle (#198)

* refactor http API handler, add test (#199)

* refactor http API handler, add test

* validate acrastruct length before decryption (#201)

* add more errors checking

* copy keys in test keystorage, add tests on encryption

* move into separate folder; add http message responses in body

* grpc handler (#200)

* change sign of comparison key length

* base part of connection handling

* save connections by descriptor

* add a little bit graceful stop service

* add grpc service

* fix log message

* fix import path

* add missing file

* validate acrastruct length before decryption

* add api test

* drop old method
fill with zeroes private key

* verbose go get

* rename api -> grpc_api

* fix network part (#203)

* fix accepting connections, network manager

* drop comment

* AcraReader http buffering (#204)

* fixing http handling

* close connection, remove buffers for http response

* add more logs to decryptor

* move defer to the top

* acra-reader -> acra-translator renaming (#205)

* acra-reader -> acra-translator

* re-gen api.pb.go

* rm old file

* Update AcraConnector to connect with AcraTranslator (#206)

* acra-reader -> acra-translator

* re-gen api.pb.go

* add separate acra-translator keys

* simplify keys check

* acra reader integration tests [T663] (#207)

* integration tests for grpc/http

* refactor serializing http response struct

* check response codes and messages in tests

* compare all messages in lower case

* add translator keystore
move filesystem keystore to separate package
run translator with own transport keys

* fix import in unit test

* return error code after failed creation of posion record

* drop keys folder before tests

* set Connection: close header to http response

* optimize response sending

* deadline for secure session handshake
timeout for http requests in tests

* refactor tests of acra-connector shutdown

* detect poison record on translator (#212)

* check poison records on acra-translator

* change path to test master key

* update log messages

cmd/acra-addzone/acra-addzone.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cossacklabs/themis/gothemis/keys"
 	log "github.com/sirupsen/logrus"
 	"os"
+	"github.com/cossacklabs/acra/keystore/filesystem"
 )
 
 // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default
@@ -61,7 +62,7 @@ func main() {
 			log.WithError(err).Errorln("can't init scell encryptor")
 			os.Exit(1)
 		}
-		keyStore, err = keystore.NewFilesystemKeyStore(output, scellEncryptor)
+		keyStore, err = filesystem.NewFilesystemKeyStore(output, scellEncryptor)
 		if err != nil {
 			log.WithError(err).Errorln("can't create key store")
 			os.Exit(1)

cmd/acra-authmanager/acra_authmanager.go

@@ -27,6 +27,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+	"github.com/cossacklabs/acra/keystore/filesystem"
 )
 
 type HashedPasswords map[string]string
@@ -51,7 +52,7 @@ func (hp HashedPasswords) Bytes() (passwordBytes []byte) {
 	return passwordBytes
 }
 
-func (hp HashedPasswords) WriteToFile(file string, keystore *keystore.FilesystemKeyStore) error {
+func (hp HashedPasswords) WriteToFile(file string, keystore *filesystem.FilesystemKeyStore) error {
 	key, err := keystore.GetAuthKey(false)
 	if err != nil {
 		return err
@@ -82,7 +83,7 @@ func (hp HashedPasswords) SetPassword(name, password string) (err error) {
 	return nil
 }
 
-func ParseHtpasswdFile(file string, keystore *keystore.FilesystemKeyStore) (passwords HashedPasswords, err error) {
+func ParseHtpasswdFile(file string, keystore *filesystem.FilesystemKeyStore) (passwords HashedPasswords, err error) {
 	htpasswdBytes, err := ioutil.ReadFile(file)
 	if err != nil {
 		return
@@ -125,7 +126,7 @@ func ParseHtpasswd(htpasswdBytes []byte) (passwords HashedPasswords, err error)
 	return
 }
 
-func RemoveUser(file, user string, keystore *keystore.FilesystemKeyStore) error {
+func RemoveUser(file, user string, keystore *filesystem.FilesystemKeyStore) error {
 	passwords, err := ParseHtpasswdFile(file, keystore)
 	if err != nil {
 		return err
@@ -138,7 +139,7 @@ func RemoveUser(file, user string, keystore *keystore.FilesystemKeyStore) error
 	return passwords.WriteToFile(file, keystore)
 }
 
-func SetPassword(file, name, password string, keystore *keystore.FilesystemKeyStore) error {
+func SetPassword(file, name, password string, keystore *filesystem.FilesystemKeyStore) error {
 	_, err := os.Stat(file)
 	passwords := HashedPasswords(map[string]string{})
 	if err == nil {
@@ -186,7 +187,7 @@ func main() {
 		log.WithError(err).Errorln("can't initialize scell encryptor")
 		os.Exit(1)
 	}
-	keyStore, err := keystore.NewFilesystemKeyStore(*keysDir, encryptor)
+	keyStore, err := filesystem.NewFilesystemKeyStore(*keysDir, encryptor)
 	if err != nil {
 		log.WithError(err).Errorln("NewFilesystemKeyStore")
 		os.Exit(1)

cmd/acra-connector/connector-mode/connector-mode.go

@@ -0,0 +1,23 @@
+package connector_mode
+
+import "strings"
+
+type ConnectorMode string
+
+const (
+	UndefinedMode ConnectorMode = "UndefinedMode"
+	AcraServerMode ConnectorMode = "AcraServer"
+	AcraTranslatorMode ConnectorMode = "AcraTranslator"
+)
+
+func CheckConnectorMode(mode string) ConnectorMode {
+	lowerCaseMode := strings.ToLower(mode)
+
+	switch lowerCaseMode {
+	case "acraserver":
+		return AcraServerMode
+	case "acratranslator":
+		return AcraTranslatorMode
+	}
+	return UndefinedMode
+}

cmd/acra-keymaker/acra-keymaker.go

@@ -22,6 +22,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"io/ioutil"
 	"os"
+	"github.com/cossacklabs/acra/keystore/filesystem"
 )
 
 // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default
@@ -31,7 +32,8 @@ var SERVICE_NAME = "acra-keymaker"
 func main() {
 	clientId := flag.String("client_id", "client", "Client id")
 	acraConnector := flag.Bool("generate_acraconnector_keys", false, "Create keypair for AcraConnector only")
-	acraserver := flag.Bool("generate_acraserver_keys", false, "Create keypair for AcraServer only")
+	acraServer := flag.Bool("generate_acraserver_keys", false, "Create keypair for AcraServer only")
+	acraTranslator := flag.Bool("generate_acratranslator_keys", false, "Create keypair for AcraTranslator only")
 	dataKeys := flag.Bool("generate_acrawriter_keys", false, "Create keypair for data encryption/decryption")
 	basicauth := flag.Bool("generate_acrawebconfig_keys", false, "Create symmetric key for AcraWebconfig's basic auth db")
 	outputDir := flag.String("keys_output_dir", keystore.DEFAULT_KEY_DIR_SHORT, "Folder where will be saved keys")
@@ -42,7 +44,7 @@ func main() {
 
 	err := cmd.Parse(DEFAULT_CONFIG_PATH, SERVICE_NAME)
 	if err != nil {
-		log.WithError(err).Errorln("can't parse args")
+		log.WithError(err).Errorln("Can't parse args")
 		os.Exit(1)
 	}
 
@@ -65,19 +67,19 @@ func main() {
 			log.Infof("You must pass master key via %v environment variable", keystore.ACRA_MASTER_KEY_VAR_NAME)
 			os.Exit(1)
 		}
-		log.WithError(err).Errorln("can't load master key")
+		log.WithError(err).Errorln("Can't load master key")
 		os.Exit(1)
 	}
 	scellEncryptor, err := keystore.NewSCellKeyEncryptor(symmetricKey)
 	if err != nil {
-		log.WithError(err).Errorln("can't init scell encryptor")
+		log.WithError(err).Errorln("Can't init scell encryptor")
 		os.Exit(1)
 	}
 	var store keystore.KeyStore
 	if *outputPublicKey != *outputDir {
-		store, err = keystore.NewFilesystemKeyStoreTwoPath(*outputDir, *outputPublicKey, scellEncryptor)
+		store, err = filesystem.NewFilesystemKeyStoreTwoPath(*outputDir, *outputPublicKey, scellEncryptor)
 	} else {
-		store, err = keystore.NewFilesystemKeyStore(*outputDir, scellEncryptor)
+		store, err = filesystem.NewFilesystemKeyStore(*outputDir, scellEncryptor)
 	}
 	if err != nil {
 		panic(err)
@@ -88,11 +90,16 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
-	} else if *acraserver {
+	} else if *acraServer {
 		err = store.GenerateServerKeys([]byte(*clientId))
 		if err != nil {
 			panic(err)
 		}
+	} else if *acraTranslator {
+		err = store.GenerateTranslatorKeys([]byte(*clientId))
+		if err != nil {
+			panic(err)
+		}
 	} else if *dataKeys {
 		err = store.GenerateDataEncryptionKeys([]byte(*clientId))
 		if err != nil {
@@ -114,6 +121,11 @@ func main() {
 			panic(err)
 		}
 
+		err = store.GenerateTranslatorKeys([]byte(*clientId))
+		if err != nil {
+			panic(err)
+		}
+
 		err = store.GenerateDataEncryptionKeys([]byte(*clientId))
 		if err != nil {
 			panic(err)

cmd/acra-poisonrecordmaker/acra-poisonrecordmaker.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cossacklabs/acra/utils"
 	log "github.com/sirupsen/logrus"
 	"os"
+	"github.com/cossacklabs/acra/keystore/filesystem"
 )
 
 // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default
@@ -52,14 +53,15 @@ func main() {
 		log.WithError(err).Errorln("can't init scell encryptor")
 		os.Exit(1)
 	}
-	store, err := keystore.NewFilesystemKeyStore(*keysDir, scellEncryptor)
+	store, err := filesystem.NewFilesystemKeyStore(*keysDir, scellEncryptor)
 	if err != nil {
 		log.WithError(err).Errorln("can't initialize key store")
 		os.Exit(1)
 	}
 	poisonRecord, err := poison.CreatePoisonRecord(store, *dataLength)
 	if err != nil {
 		log.WithError(err).Errorln("can't create poison record")
+		os.Exit(1)
 	}
 	fmt.Println(base64.StdEncoding.EncodeToString(poisonRecord))
 }

cmd/acra-rollback/acra-rollback.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
+	"github.com/cossacklabs/acra/keystore/filesystem"
 )
 
 // DEFAULT_CONFIG_PATH relative path to config which will be parsed as default
@@ -230,7 +231,7 @@ func main() {
 		log.WithError(err).Errorln("can't init scell encryptor")
 		os.Exit(1)
 	}
-	keystorage, err := keystore.NewFilesystemKeyStore(absKeysDir, scellEncryptor)
+	keystorage, err := filesystem.NewFilesystemKeyStore(absKeysDir, scellEncryptor)
 	if err != nil {
 		log.WithError(err).Errorln("can't create key store")
 		os.Exit(1)

cmd/acra-server/acra-server.go

@@ -29,6 +29,7 @@ import (
 	"github.com/cossacklabs/acra/network"
 	"github.com/cossacklabs/acra/utils"
 	log "github.com/sirupsen/logrus"
+	"github.com/cossacklabs/acra/keystore/filesystem"
 )
 
 var restartSignalsChannel chan os.Signal
@@ -192,7 +193,7 @@ func main() {
 		log.WithError(err).Errorln("can't init scell encryptor")
 		os.Exit(1)
 	}
-	keyStore, err := keystore.NewFilesystemKeyStore(*keysDir, scellEncryptor)
+	keyStore, err := filesystem.NewFilesystemKeyStore(*keysDir, scellEncryptor)
 	if err != nil {
 		log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantInitKeyStore).
 			Errorln("Can't initialise keystore")

cmd/acra-server/listener.go

@@ -232,11 +232,11 @@ func (server *SServer) StartFromFileDescriptor(fd uintptr) {
 		if err != nil {
 			if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
 				log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorConnectionDroppedByTimeout).
-					Errorf("Stop accepting new connections", connection)
+					Errorf("Stop accepting new connections")
 				return
 			}
 			log.WithError(err).WithField(logging.FieldKeyEventCode, logging.EventCodeErrorCantAcceptNewConnections).
-				Errorf("Can't accept new connection (connection=%v)", connection)
+				Errorf("Can't accept new connection")
 			continue
 		}
 		// unix socket and value == '@'
@@ -253,16 +253,9 @@ func (server *SServer) StartFromFileDescriptor(fd uintptr) {
 	}
 }
 
-// deadlineListener is extended net.Listener interface with SetDeadline method that added for abstraction of calling
-// SetDeadline between two listener types (TcpListener and UnixListener) that support this method
-type deadlineListener interface {
-	net.Listener
-	SetDeadline(t time.Time) error
-}
-
 // stopAcceptConnections stop accepting by setting deadline and then background code that call Accept will took error and
 // stop execution
-func stopAcceptConnections(listener deadlineListener) (err error) {
+func stopAcceptConnections(listener network.DeadlineListener) (err error) {
 	if listener != nil {
 		err = listener.SetDeadline(time.Now())
 		if err != nil {
@@ -281,38 +274,21 @@ func stopAcceptConnections(listener deadlineListener) (err error) {
 
 func (server *SServer) StopListeners() {
 	var err error
-	var listener deadlineListener
+	var deadlineListener network.DeadlineListener
 	log.Debugln("Stopping listeners")
 
-	switch server.listenerACRA.(type) {
-	case *net.TCPListener:
-		listener = server.listenerACRA.(*net.TCPListener)
-	case *net.UnixListener:
-		listener = server.listenerACRA.(*net.UnixListener)
-	case nil:
-		log.Debugln("hasn't acra listener")
-	default:
-		log.Warningln("unsupported listener")
-	}
+	for _, listener := range server.listeners {
 
-	if err = stopAcceptConnections(listener); err != nil {
-		log.WithError(err).Warningln("can't set deadline for server listener")
-	}
+		deadlineListener, err = network.CastListenerToDeadline(listener)
+		if err != nil {
+			log.WithError(err).Warningln("Can't cast listener")
+			continue
+		}
 
-	switch server.listenerAPI.(type) {
-	case *net.TCPListener:
-		listener = server.listenerACRA.(*net.TCPListener)
-	case *net.UnixListener:
-		listener = server.listenerACRA.(*net.UnixListener)
-	case nil:
-		log.Debugln("hasn't api listener")
-	default:
-		log.Warningln("unsupported listener")
-	}
-	if err = stopAcceptConnections(listener); err != nil {
-		log.WithError(err).Warningln("can't set deadline for api listener")
+		if err = stopAcceptConnections(deadlineListener); err != nil {
+			log.WithError(err).Warningln("Can't set deadline for listener")
+		}
 	}
-
 }
 
 func (server *SServer) WaitConnections(duration time.Duration) {