From 117b287e33b6e70fdaf790d7d7bf8473ad09de56 Mon Sep 17 00:00:00 2001 From: iknite Date: Sat, 22 Dec 2018 17:11:59 +0100 Subject: [PATCH] Finish TLS configuration and tests --- .travis.yml | 2 ++ api/apihttp/apihttp.go | 12 ++--------- client/client.go | 38 +++++++++++++++++++++++++---------- client/client_test.go | 4 ++-- client/config.go | 14 ++++++------- cmd/client.go | 25 +++++++++++------------ cmd/client_add.go | 5 ++--- cmd/client_incremental.go | 7 +++---- cmd/client_membership.go | 12 +++++------ cmd/context.go | 6 +++--- gossip/auditor/auditor.go | 4 ++-- gossip/monitor/monitor.go | 4 ++-- server/server.go | 2 +- tests/e2e/add_verify_test.go | 2 +- tests/e2e/agents_test.go | 2 +- tests/e2e/cli_test.go | 15 +++++--------- tests/e2e/incremental_test.go | 2 +- tests/e2e/setup.go | 29 ++++++++++---------------- 18 files changed, 89 insertions(+), 96 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27d562573..4284a7733 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,8 @@ notifications: before_script: - ssh-keygen -t ed25519 -f $HOME/.ssh/id_ed25519 -P '' + - openssl genrsa -out $HOME/.ssh/server.key 2048 + - openssl req -new -x509 -sha256 -key $HOME/.ssh/server.key -out $HOME/.ssh/server.crt -days 3650 - go mod download script: diff --git a/api/apihttp/apihttp.go b/api/apihttp/apihttp.go index fdb5f8248..d773d2a42 100644 --- a/api/apihttp/apihttp.go +++ b/api/apihttp/apihttp.go @@ -105,6 +105,7 @@ func Add(balloon raftwal.RaftBalloonApi) http.HandlerFunc { http.Error(w, err.Error(), http.StatusInternalServerError) return } + snapshot := &protocol.Snapshot{ response.HistoryDigest, response.HyperDigest, @@ -120,6 +121,7 @@ func Add(balloon raftwal.RaftBalloonApi) http.HandlerFunc { w.WriteHeader(http.StatusCreated) w.Write(out) + return } @@ -344,13 +346,3 @@ func LogHandler(handle http.Handler) http.HandlerFunc { } } } - -// STSHandler adds TLS Header to the handlers -func STSHandler(handle http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, request *http.Request) { - w.Header().Add( - "Strict-Transport-Security", - "max-age=63072000; includeSubDomains", - ) - } -} diff --git a/client/client.go b/client/client.go index b38ab1e4f..34f8dcc73 100644 --- a/client/client.go +++ b/client/client.go @@ -25,7 +25,9 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" + "net/url" "time" "github.com/bbva/qed/balloon" @@ -37,22 +39,32 @@ import ( type HTTPClient struct { conf *Config - http.Client + *http.Client } // NewHTTPClient will return a new instance of HTTPClient. -func NewHTTPClient(conf *Config) *HTTPClient { - var c http.Client - if conf.EnableTLS { - c = http.Client{ +func NewHTTPClient(conf Config) *HTTPClient { + var tlsConf *tls.Config + + if conf.Insecure { + tlsConf = &tls.Config{InsecureSkipVerify: true} + } else { + tlsConf = &tls.Config{} + } + + return &HTTPClient{ + &conf, + &http.Client{ + Timeout: time.Second * 10, Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + Dial: (&net.Dialer{ + Timeout: 5 * time.Second, + }).Dial, + TLSClientConfig: tlsConf, + TLSHandshakeTimeout: 5 * time.Second, }, - } - } else { - c = http.Client{} + }, } - return &HTTPClient{conf, c} } @@ -77,8 +89,12 @@ func (c HTTPClient) exponentialBackoff(req *http.Request) (*http.Response, error } func (c HTTPClient) doReq(method, path string, data []byte) ([]byte, error) { + url, err := url.Parse(c.conf.Endpoint + path) + if err != nil { + panic(err) + } - req, err := http.NewRequest(method, c.conf.Endpoint+path, bytes.NewBuffer(data)) + req, err := http.NewRequest(method, fmt.Sprintf("%s", url), bytes.NewBuffer(data)) if err != nil { panic(err) } diff --git a/client/client_test.go b/client/client_test.go index 0cd868456..8d1fbde0c 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -42,10 +42,10 @@ func init() { func setup() func() { mux = http.NewServeMux() server = httptest.NewServer(mux) - client = NewHTTPClient(&Config{ + client = NewHTTPClient(Config{ Endpoint: server.URL, APIKey: "my-awesome-api-key", - EnableTLS: false, + Insecure: false, }) return func() { server.Close() diff --git a/client/config.go b/client/config.go index fc611a45d..98e38926c 100644 --- a/client/config.go +++ b/client/config.go @@ -17,20 +17,20 @@ package client type Config struct { - // Server host:port to consult + // Server host:port to consult. Endpoint string - // ApiKey to query the server endpoint + // ApiKey to query the server endpoint. APIKey string - // Enable TLS service - EnableTLS bool + // Enable self-signed certificates, allowing MiTM vector attacks. + Insecure bool } func DefaultConfig() *Config { return &Config{ - Endpoint: "localhost:8080", - APIKey: "my-key", - EnableTLS: true, + Endpoint: "localhost:8080", + APIKey: "my-key", + Insecure: true, } } diff --git a/cmd/client.go b/cmd/client.go index a3a00cc1b..4a05e590b 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -23,14 +23,7 @@ import ( ) func newClientCommand(ctx *cmdContext) *cobra.Command { - var endpoint string - var disableTLS bool - - client := client.NewHTTPClient(&client.Config{ - Endpoint: endpoint, - APIKey: ctx.apiKey, - EnableTLS: !disableTLS, - }) + clientCtx := &clientContext{} cmd := &cobra.Command{ Use: "client", @@ -38,16 +31,22 @@ func newClientCommand(ctx *cmdContext) *cobra.Command { Long: `Client process for emitting events to a qed server`, PersistentPreRun: func(cmd *cobra.Command, args []string) { log.SetLogger("QedClient", ctx.logLevel) + + clientCtx.client = client.NewHTTPClient(client.Config{ + Endpoint: clientCtx.endpoint, + APIKey: ctx.apiKey, + Insecure: clientCtx.insecure, + }) }, TraverseChildren: true, } - cmd.PersistentFlags().StringVarP(&endpoint, "endpoint", "e", "localhost:8080", "Endpoint for REST requests on (host:port)") - cmd.PersistentFlags().BoolVar(&disableTLS, "insecure", false, "Disable TLS transport") + cmd.PersistentFlags().StringVarP(&clientCtx.endpoint, "endpoint", "e", "localhost:8080", "Endpoint for REST requests on (host:port)") + cmd.PersistentFlags().BoolVar(&clientCtx.insecure, "insecure", false, "Disable TLS transport") - cmd.AddCommand(newAddCommand(client)) - cmd.AddCommand(newMembershipCommand(client)) - cmd.AddCommand(newIncrementalCommand(client)) + cmd.AddCommand(newAddCommand(clientCtx)) + cmd.AddCommand(newMembershipCommand(clientCtx)) + cmd.AddCommand(newIncrementalCommand(clientCtx)) return cmd } diff --git a/cmd/client_add.go b/cmd/client_add.go index 3145e684a..64cb9068e 100644 --- a/cmd/client_add.go +++ b/cmd/client_add.go @@ -19,11 +19,10 @@ package cmd import ( "github.com/spf13/cobra" - "github.com/bbva/qed/client" "github.com/bbva/qed/log" ) -func newAddCommand(client *client.HTTPClient) *cobra.Command { +func newAddCommand(ctx *clientContext) *cobra.Command { var key, value string @@ -34,7 +33,7 @@ func newAddCommand(client *client.HTTPClient) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { log.Infof("Adding key [ %s ] with value [ %s ]\n", key, value) - snapshot, err := client.Add(key) + snapshot, err := ctx.client.Add(key) if err != nil { return err } diff --git a/cmd/client_incremental.go b/cmd/client_incremental.go index e886e41a0..d9bcb127d 100644 --- a/cmd/client_incremental.go +++ b/cmd/client_incremental.go @@ -19,7 +19,6 @@ package cmd import ( "encoding/hex" - "github.com/bbva/qed/client" "github.com/bbva/qed/hashing" "github.com/bbva/qed/protocol" @@ -28,7 +27,7 @@ import ( "github.com/bbva/qed/log" ) -func newIncrementalCommand(client *client.HTTPClient) *cobra.Command { +func newIncrementalCommand(ctx *clientContext) *cobra.Command { var start, end uint64 var verify bool @@ -53,7 +52,7 @@ func newIncrementalCommand(client *client.HTTPClient) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { log.Infof("Querying incremental between versions [ %d ] and [ %d ]\n", start, end) - proof, err := client.Incremental(start, end) + proof, err := ctx.client.Incremental(start, end) if err != nil { return err } @@ -68,7 +67,7 @@ func newIncrementalCommand(client *client.HTTPClient) *cobra.Command { log.Infof("Verifying with snapshots: \n\tStartDigest: %s\n\tEndDigest: %s\n", startDigest, endDigest) - if client.VerifyIncremental(proof, startSnapshot, endSnapshot, hashing.NewSha256Hasher()) { + if ctx.client.VerifyIncremental(proof, startSnapshot, endSnapshot, hashing.NewSha256Hasher()) { log.Info("Verify: OK") } else { log.Info("Verify: KO") diff --git a/cmd/client_membership.go b/cmd/client_membership.go index c1d4f0d4c..dd51914d2 100644 --- a/cmd/client_membership.go +++ b/cmd/client_membership.go @@ -19,16 +19,14 @@ package cmd import ( "encoding/hex" - "github.com/bbva/qed/client" - "github.com/bbva/qed/hashing" - "github.com/bbva/qed/protocol" - "github.com/spf13/cobra" + "github.com/bbva/qed/hashing" "github.com/bbva/qed/log" + "github.com/bbva/qed/protocol" ) -func newMembershipCommand(client *client.HTTPClient) *cobra.Command { +func newMembershipCommand(ctx *clientContext) *cobra.Command { hasherF := hashing.NewSha256Hasher var version uint64 @@ -68,7 +66,7 @@ func newMembershipCommand(client *client.HTTPClient) *cobra.Command { digest, _ = hex.DecodeString(eventDigest) } - membershipResult, err = client.MembershipDigest(digest, version) + membershipResult, err = ctx.client.MembershipDigest(digest, version) if err != nil { return err } @@ -99,7 +97,7 @@ func newMembershipCommand(client *client.HTTPClient) *cobra.Command { log.Infof("Verifying with Snapshot: \n\tEventDigest:%x\n\tHyperDigest: %s\n\tHistoryDigest: %s\n\tVersion: %d\n", digest, hyperDigest, historyDigest, version) - if client.DigestVerify(membershipResult, snapshot, hasherF) { + if ctx.client.DigestVerify(membershipResult, snapshot, hasherF) { log.Info("Verify: OK") } else { log.Info("Verify: KO") diff --git a/cmd/context.go b/cmd/context.go index beffbed6e..36c9abfc3 100644 --- a/cmd/context.go +++ b/cmd/context.go @@ -26,9 +26,9 @@ type cmdContext struct { } type clientContext struct { - endpoint string - disableTLS bool - client *client.HTTPClient + endpoint string + insecure bool + client *client.HTTPClient } type agentContext struct { diff --git a/gossip/auditor/auditor.go b/gossip/auditor/auditor.go index 1d0fb6ae1..bab1044a7 100644 --- a/gossip/auditor/auditor.go +++ b/gossip/auditor/auditor.go @@ -56,10 +56,10 @@ type Auditor struct { func NewAuditor(conf Config) (*Auditor, error) { auditor := Auditor{ - qed: client.NewHTTPClient(&client.Config{ + qed: client.NewHTTPClient(client.Config{ Endpoint: conf.QEDUrls[0], APIKey: conf.APIKey, - EnableTLS: false, + Insecure: false, }), conf: conf, taskCh: make(chan Task, 100), diff --git a/gossip/monitor/monitor.go b/gossip/monitor/monitor.go index e63bdddd0..bb167daaa 100644 --- a/gossip/monitor/monitor.go +++ b/gossip/monitor/monitor.go @@ -57,10 +57,10 @@ type Monitor struct { func NewMonitor(conf Config) (*Monitor, error) { monitor := Monitor{ - client: client.NewHTTPClient(&client.Config{ + client: client.NewHTTPClient(client.Config{ Endpoint: conf.QedUrls[0], APIKey: conf.APIKey, - EnableTLS: false, + Insecure: false, }), conf: conf, taskCh: make(chan QueryTask, 100), diff --git a/server/server.go b/server/server.go index 16c92fc7e..0684a6806 100644 --- a/server/server.go +++ b/server/server.go @@ -313,7 +313,7 @@ func newTLSServer(addr string, mux *http.ServeMux) *http.Server { return &http.Server{ Addr: addr, - Handler: apihttp.STSHandler(apihttp.LogHandler(mux)), + Handler: apihttp.LogHandler(mux), TLSConfig: cfg, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0), } diff --git a/tests/e2e/add_verify_test.go b/tests/e2e/add_verify_test.go index e2a2aa49c..5e576c116 100644 --- a/tests/e2e/add_verify_test.go +++ b/tests/e2e/add_verify_test.go @@ -28,7 +28,7 @@ import ( ) func TestAddVerify(t *testing.T) { - before, after := setupServer(0, "", t) + before, after := setupServer(0, "", false, t) scenario, let := scope.Scope(t, before, after) client := getClient(0) diff --git a/tests/e2e/agents_test.go b/tests/e2e/agents_test.go index ab20a539c..537210edc 100644 --- a/tests/e2e/agents_test.go +++ b/tests/e2e/agents_test.go @@ -66,7 +66,7 @@ func getAlert() ([]byte, error) { func TestAgents(t *testing.T) { bStore, aStore := setupStore(t) - bServer, aServer := setupServer(0, "", t) + bServer, aServer := setupServer(0, "", false, t) bAuditor, aAuditor := setupAuditor(0, t) bMonitor, aMonitor := setupMonitor(0, t) bPublisher, aPublisher := setupPublisher(0, t) diff --git a/tests/e2e/cli_test.go b/tests/e2e/cli_test.go index d83a44f43..f614a7d23 100644 --- a/tests/e2e/cli_test.go +++ b/tests/e2e/cli_test.go @@ -28,7 +28,7 @@ import ( ) func TestCli(t *testing.T) { - before, after := setupServer(0, "", t) + before, after := setupServer(0, "", true, t) scenario, let := scope.Scope(t, before, merge(after)) scenario("Add one event through cli and verify it", func() { @@ -39,7 +39,7 @@ func TestCli(t *testing.T) { "./../../main.go", fmt.Sprintf("--apikey=%s", APIKey), "client", - fmt.Sprintf("--endpoint=%s", QEDUrl), + fmt.Sprintf("--endpoint=%s", QEDTLS), "add", "--key='test event'", "--value=2", @@ -47,9 +47,7 @@ func TestCli(t *testing.T) { "--insecure", ) - o, err := cmd.CombinedOutput() - - fmt.Printf(">>>>> %s", o) + _, err := cmd.CombinedOutput() assert.NoError(t, err, "Subprocess must not exit with status 1") }) @@ -60,7 +58,7 @@ func TestCli(t *testing.T) { "./../../main.go", fmt.Sprintf("--apikey=%s", APIKey), "client", - fmt.Sprintf("--endpoint=%s", QEDUrl), + fmt.Sprintf("--endpoint=%s", QEDTLS), "membership", "--hyperDigest=81ae2d8f6ecec9c5837d12a09e3b42a1c880b6c77f81ff1f85aef36dac4fdf6a", "--historyDigest=0f5129eaf5dbfb1405ff072a04d716aaf4e4ba4247a3322c41582e970dbb7b00", @@ -75,17 +73,15 @@ func TestCli(t *testing.T) { assert.NoError(t, err, "Subprocess must not exit with status 1") assert.True(t, strings.Contains(fmt.Sprintf("%s", stdoutStderr), "Verify: OK"), "Must verify with eventDigest") - }) let("verify event with eventDigest", func(t *testing.T) { - cmd := exec.Command("go", "run", "./../../main.go", fmt.Sprintf("--apikey=%s", APIKey), "client", - fmt.Sprintf("--endpoint=%s", QEDUrl), + fmt.Sprintf("--endpoint=%s", QEDTLS), "membership", "--hyperDigest=81ae2d8f6ecec9c5837d12a09e3b42a1c880b6c77f81ff1f85aef36dac4fdf6a", "--historyDigest=0f5129eaf5dbfb1405ff072a04d716aaf4e4ba4247a3322c41582e970dbb7b00", @@ -100,7 +96,6 @@ func TestCli(t *testing.T) { assert.NoError(t, err, "Subprocess must not exit with status 1") assert.True(t, strings.Contains(fmt.Sprintf("%s", stdoutStderr), "Verify: OK"), "Must verify with eventDigest") - }) }) diff --git a/tests/e2e/incremental_test.go b/tests/e2e/incremental_test.go index 27637b6e4..efa030d39 100644 --- a/tests/e2e/incremental_test.go +++ b/tests/e2e/incremental_test.go @@ -28,7 +28,7 @@ import ( ) func TestIncrementalConsistency(t *testing.T) { - before, after := setupServer(0, "", t) + before, after := setupServer(0, "", false, t) scenario, let := scope.Scope(t, before, after) client := getClient(0) diff --git a/tests/e2e/setup.go b/tests/e2e/setup.go index a592457b4..d6cd9ba1b 100644 --- a/tests/e2e/setup.go +++ b/tests/e2e/setup.go @@ -29,30 +29,21 @@ import ( "github.com/bbva/qed/gossip/member" "github.com/bbva/qed/gossip/monitor" "github.com/bbva/qed/gossip/publisher" - "github.com/bbva/qed/log" "github.com/bbva/qed/server" "github.com/bbva/qed/testutils/scope" ) const ( QEDUrl = "http://127.0.0.1:8080" + QEDTLS = "https://localhost:8080" QEDGossip = "127.0.0.1:9010" - QEDTamperURL = "http://localhost:8081/tamper" + QEDTamperURL = "http://127.0.0.1:8081/tamper" StoreURL = "http://127.0.0.1:8888" APIKey = "my-key" cacheSize = 50000 storageType = "badger" ) -var keyFile string - -func init() { - usr, _ := user.Current() - keyFile = fmt.Sprintf("%s/.ssh/id_ed25519", usr.HomeDir) - - log.SetLogger("", log.SILENT) -} - // merge function is a helper function that execute all the variadic parameters // inside a score.TestF function func merge(list ...scope.TestF) scope.TestF { @@ -191,11 +182,13 @@ func setupStore(t *testing.T) (scope.TestF, scope.TestF) { return before, after } -func setupServer(id int, joinAddr string, t *testing.T) (scope.TestF, scope.TestF) { +func setupServer(id int, joinAddr string, tls bool, t *testing.T) (scope.TestF, scope.TestF) { var srv *server.Server var err error path := fmt.Sprintf("/var/tmp/e2e-qed%d/", id) + usr, _ := user.Current() + before := func(t *testing.T) { os.RemoveAll(path) err = os.MkdirAll(path, os.FileMode(0755)) @@ -212,10 +205,10 @@ func setupServer(id int, joinAddr string, t *testing.T) (scope.TestF, scope.Test conf.GossipAddr = fmt.Sprintf("127.0.0.1:901%d", id) conf.DBPath = path + "data" conf.RaftPath = path + "raft" - conf.PrivateKeyPath = keyFile + conf.PrivateKeyPath = fmt.Sprintf("%s/.ssh/id_ed25519", usr.HomeDir) conf.EnableProfiling = true conf.EnableTampering = true - conf.EnableTLS = false + conf.EnableTLS = tls fmt.Printf("Server config: %+v\n", conf) @@ -251,9 +244,9 @@ func endPoint(id int) string { } func getClient(id int) *client.HTTPClient { - return client.NewHTTPClient(&client.Config{ - Endpoint: endPoint(id), - APIKey: APIKey, - EnableTLS: false, + return client.NewHTTPClient(client.Config{ + Endpoint: endPoint(id), + APIKey: APIKey, + Insecure: false, }) }