diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml index d74c9c2e..e0ba32e5 100644 --- a/.github/workflows/test_and_release.yml +++ b/.github/workflows/test_and_release.yml @@ -40,7 +40,7 @@ jobs: docker build . -t bbvalabsci/kapow-spec-test-suite:latest - name: Spec test run: | - docker run --mount type=bind,source=$(pwd)/build/kapow,target=/usr/local/bin/kapow bbvalabsci/kapow-spec-test-suite:latest behave --tags=~@skip + docker run --mount type=bind,source=$(pwd)/build/kapow,target=/usr/bin/kapow bbvalabsci/kapow-spec-test-suite:latest "behave --tags=~@skip" doc-test: runs-on: ubuntu-20.04 steps: diff --git a/.gitignore b/.gitignore index 372dd5d0..bf081ccc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ build docs/build docs/Pipfile.lock + +node_modules + +*.swp diff --git a/Makefile b/Makefile index 4a22e19c..ef53067d 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,8 @@ coverage: test race install: build CGO_ENABLED=0 $(GOINSTALL) ./... -acceptance: install - make -C ./spec/test +acceptance: build + cd ./spec/test && PATH=$(PWD)/build:$$PATH nix-shell --command make deps: @echo "deps here" diff --git a/README.md b/README.md index 9665faf3..4f3ecf49 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,9 @@ You can find the complete documentation and examples [here](https://kapow.readth ## Security -Please consider the following security caveats **before** using *Kapow!* - -- [Issue #119](https://github.com/BBVA/kapow/issues/119) -- [Security Concerns](https://kapow.readthedocs.io/en/stable/the_project/security.html#security-concerns) +Please consider the following +[Security Concerns](https://kapow.readthedocs.io/en/stable/the_project/security.html#security-concerns) +**before** using *Kapow!* If you are not 100% sure about what you are doing we recommend not using *Kapow!* diff --git a/docs/source/concepts/interfaces.rst b/docs/source/concepts/interfaces.rst index b84eac17..d104b2dc 100644 --- a/docs/source/concepts/interfaces.rst +++ b/docs/source/concepts/interfaces.rst @@ -16,16 +16,20 @@ By default it binds to address ``0.0.0.0`` and port ``8080``, but that can be changed via the ``--bind`` flag. -.. _http-control-interface: +.. _https-control-interface: -HTTP Control Interface ----------------------- +HTTPS Control Interface +----------------------- -The `HTTP Control Interface` is used by the command ``kapow route`` to +The `HTTPS Control Interface` is used by the command ``kapow route`` to administer the list of system routes. +This interface uses mTLS by default (double-pinned autogenerated certs). + By default it binds to address ``127.0.0.1`` and port ``8081``, but that can be -changed via the ``--control-bind`` flag. +changed via the ``--control-bind`` flag. If this is the case, consider +also ``--control-reachable-addr`` which will configure the autogenerated +certificate to match that address. .. _http-data-interface: diff --git a/docs/source/concepts/request_life_cycle.rst b/docs/source/concepts/request_life_cycle.rst index d3729ba0..d2976a95 100644 --- a/docs/source/concepts/request_life_cycle.rst +++ b/docs/source/concepts/request_life_cycle.rst @@ -30,8 +30,8 @@ The spawned entrypoint is run with the following variables added to its environment: - :envvar:`KAPOW_HANDLER_ID`: Containing the `HANDLER_ID` -- :envvar:`KAPOW_DATAAPI_URL`: With the URL of the :ref:`http-data-interface` -- :envvar:`KAPOW_CONTROLAPI_URL`: With the URL of the :ref:`http-control-interface` +- :envvar:`KAPOW_DATA_URL`: With the URL of the :ref:`http-data-interface` +- :envvar:`KAPOW_CONTROL_URL`: With the URL of the :ref:`https-control-interface` 3. ``kapow set /response/body banana`` diff --git a/docs/source/examples/managing_routes.rst b/docs/source/examples/managing_routes.rst index 299273ce..b3d530e9 100644 --- a/docs/source/examples/managing_routes.rst +++ b/docs/source/examples/managing_routes.rst @@ -100,7 +100,7 @@ Or, if you want human-readable output, you can use :program:`jq`: .. note:: - *Kapow!* has a :ref:`http-control-interface`, bound by default to + *Kapow!* has a :ref:`https-control-interface`, bound by default to ``localhost:8081``. diff --git a/internal/certs/certs.go b/internal/certs/certs.go new file mode 100644 index 00000000..24ebc9f1 --- /dev/null +++ b/internal/certs/certs.go @@ -0,0 +1,99 @@ +package certs + +import ( + "bytes" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "time" + + "github.com/BBVA/kapow/internal/logger" +) + +type Cert struct { + X509Cert *x509.Certificate + PrivKey crypto.PrivateKey + SignedCert []byte +} + +func (c Cert) SignedCertPEMBytes() []byte { + + PEM := new(bytes.Buffer) + err := pem.Encode(PEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: c.SignedCert, + }) + if err != nil { + logger.L.Fatal(err) + } + + return PEM.Bytes() +} + +func (c Cert) PrivateKeyPEMBytes() []byte { + PEM := new(bytes.Buffer) + err := pem.Encode(PEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(c.PrivKey.(*rsa.PrivateKey)), + }) + if err != nil { + logger.L.Fatal(err) + } + + return PEM.Bytes() +} + +func GenCert(name, altName string, isServer bool) Cert { + + usage := x509.ExtKeyUsageClientAuth + if isServer { + usage = x509.ExtKeyUsageServerAuth + } + + var dnsNames []string + var ipAddresses []net.IP + if altName != "" { + if ipAddr := net.ParseIP(altName); ipAddr != nil { + ipAddresses = []net.IP{ipAddr} + } else { + dnsNames = []string{altName} + } + } + + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1), + DNSNames: dnsNames, + IPAddresses: ipAddresses, + Subject: pkix.Name{ + CommonName: name, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: false, + BasicConstraintsValid: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + usage, + }, + } + + certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + logger.L.Fatal(err) + } + + certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivKey.PublicKey, certPrivKey) + if err != nil { + logger.L.Fatal(err) + } + + return Cert{ + X509Cert: cert, + PrivKey: certPrivKey, + SignedCert: certBytes, + } +} diff --git a/internal/client/client_test.go b/internal/client/client_test.go new file mode 100644 index 00000000..9bbb6d1d --- /dev/null +++ b/internal/client/client_test.go @@ -0,0 +1,29 @@ +/* + * Copyright 2019 Banco Bilbao Vizcaya Argentaria, S.A. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "os" + "testing" + + "github.com/BBVA/kapow/internal/http" +) + +func TestMain(m *testing.M) { + http.ControlClientGenerator = nil + os.Exit(m.Run()) +} diff --git a/internal/client/get.go b/internal/client/get.go index f0816eed..38cee47f 100644 --- a/internal/client/get.go +++ b/internal/client/get.go @@ -25,5 +25,5 @@ import ( // GetData will perform the request and write the results on the provided writer func GetData(host, id, path string, w io.Writer) error { url := host + "/handlers/" + id + path - return http.Get(url, "", nil, w) + return http.Get(url, nil, w, nil) } diff --git a/internal/client/route_add.go b/internal/client/route_add.go index b79e8a05..bde6c576 100644 --- a/internal/client/route_add.go +++ b/internal/client/route_add.go @@ -36,5 +36,5 @@ func AddRoute(host, path, method, entrypoint, command string, w io.Writer) error payload["entrypoint"] = entrypoint } body, _ := json.Marshal(payload) - return http.Post(url, "application/json", bytes.NewReader(body), w) + return http.Post(url, bytes.NewReader(body), w, http.ControlClientGenerator, http.AsJSON) } diff --git a/internal/client/route_list.go b/internal/client/route_list.go index a22f86fa..d157a559 100644 --- a/internal/client/route_list.go +++ b/internal/client/route_list.go @@ -25,5 +25,5 @@ import ( // ListRoutes queries the kapow! instance for the routes that are registered func ListRoutes(host string, w io.Writer) error { url := host + "/routes" - return http.Get(url, "", nil, w) + return http.Get(url, nil, w, http.ControlClientGenerator) } diff --git a/internal/client/route_remove.go b/internal/client/route_remove.go index 53329ed6..283e7fa1 100644 --- a/internal/client/route_remove.go +++ b/internal/client/route_remove.go @@ -23,5 +23,5 @@ import ( // RemoveRoute removes a registered route in Kapow! server func RemoveRoute(host, id string) error { url := host + "/routes/" + id - return http.Delete(url, "", nil, nil) + return http.Delete(url, nil, nil, http.ControlClientGenerator) } diff --git a/internal/client/set.go b/internal/client/set.go index 745c1aba..33846cb8 100644 --- a/internal/client/set.go +++ b/internal/client/set.go @@ -24,5 +24,5 @@ import ( func SetData(host, handlerID, path string, r io.Reader) error { url := host + "/handlers/" + handlerID + path - return http.Put(url, "", r, nil) + return http.Put(url, r, nil, nil) } diff --git a/internal/cmd/route.go b/internal/cmd/route.go index e6310af6..1fc0e0f4 100644 --- a/internal/cmd/route.go +++ b/internal/cmd/route.go @@ -43,7 +43,7 @@ func init() { } }, } - routeListCmd.Flags().String("control-url", getEnv("KAPOW_CONTROL_URL", "http://localhost:8081"), "Kapow! control interface URL") + routeListCmd.Flags().String("control-url", getEnv("KAPOW_CONTROL_URL", "https://localhost:8081"), "Kapow! control interface URL") // TODO: Manage args for url_pattern and command_file (2 exact args) var routeAddCmd = &cobra.Command{ @@ -78,7 +78,7 @@ func init() { }, } // TODO: Add default values for flags and remove path flag - routeAddCmd.Flags().String("control-url", getEnv("KAPOW_CONTROL_URL", "http://localhost:8081"), "Kapow! control interface URL") + routeAddCmd.Flags().String("control-url", getEnv("KAPOW_CONTROL_URL", "https://localhost:8081"), "Kapow! control interface URL") routeAddCmd.Flags().StringP("method", "X", "GET", "HTTP method to accept") routeAddCmd.Flags().StringP("entrypoint", "e", "", "Command to execute") routeAddCmd.Flags().StringP("command", "c", "", "Command to pass to the shell") @@ -95,7 +95,7 @@ func init() { } }, } - routeRemoveCmd.Flags().String("control-url", getEnv("KAPOW_CONTROL_URL", "http://localhost:8081"), "Kapow! control interface URL") + routeRemoveCmd.Flags().String("control-url", getEnv("KAPOW_CONTROL_URL", "https://localhost:8081"), "Kapow! control interface URL") RouteCmd.AddCommand(routeListCmd) RouteCmd.AddCommand(routeAddCmd) diff --git a/internal/cmd/server.go b/internal/cmd/server.go index df2efcd4..e30963cd 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -19,16 +19,40 @@ package cmd import ( "bufio" "errors" + "fmt" "io" "os" + "strings" "sync" "github.com/spf13/cobra" + "github.com/BBVA/kapow/internal/certs" "github.com/BBVA/kapow/internal/logger" "github.com/BBVA/kapow/internal/server" ) +func banner() { + fmt.Fprintln(os.Stderr, ` + %% %%%% + %%% %%% + %% %%% %%% + %%%%%%% %%% %%% %%% %%% + *%% %%%%%%%%%%%%%%% %%%% %%% %% + %% %%%%%%%%%. %%% %%%% %%% %%%%%%%% + %%%% %%% %%% %%% %%% %%%%%% %%%% + %%% %%% %%%%%% %%% %%%% %%% %%%% %%%% %%% %%% + %%% %%% %% %%% %%%%% %%%%% %%%% %%% + %%% %%% %% %%%%%%%%% %%%%%%%%%% + %%%%%% %%% %%%%%% %%% + %%% %%%%% %% %%%%%% + %%% %%%%%%% + %%%% + % If you can script it, you can HTTP it. + + `) +} + // ServerCmd is the command line interface for kapow server var ServerCmd = &cobra.Command{ Use: "server [optional flags] [optional init program(s)]", @@ -42,6 +66,8 @@ var ServerCmd = &cobra.Command{ sConf.ControlBindAddr, _ = cmd.Flags().GetString("control-bind") sConf.DataBindAddr, _ = cmd.Flags().GetString("data-bind") + controlReachableAddr, _ := cmd.Flags().GetString("control-reachable-addr") + sConf.CertFile, _ = cmd.Flags().GetString("certfile") sConf.KeyFile, _ = cmd.Flags().GetString("keyfile") @@ -49,29 +75,51 @@ var ServerCmd = &cobra.Command{ sConf.ClientCaFile, _ = cmd.Flags().GetString("clientcafile") sConf.Debug, _ = cmd.Flags().GetBool("debug") + sConf.ControlServerCert = certs.GenCert("control_server", extractHost(controlReachableAddr), true) + sConf.ControlClientCert = certs.GenCert("control_client", "", false) + // Set environment variables KAPOW_DATA_URL and KAPOW_CONTROL_URL only if they aren't set so we don't overwrite user's preferences if _, exist := os.LookupEnv("KAPOW_DATA_URL"); !exist { os.Setenv("KAPOW_DATA_URL", "http://"+sConf.DataBindAddr) } if _, exist := os.LookupEnv("KAPOW_CONTROL_URL"); !exist { - os.Setenv("KAPOW_CONTROL_URL", "http://"+sConf.ControlBindAddr) + os.Setenv("KAPOW_CONTROL_URL", "https://"+controlReachableAddr) } + banner() server.StartServer(sConf) for _, path := range args { - go Run(path, sConf.Debug) + go Run( + path, + sConf.Debug, + sConf.ControlServerCert.SignedCertPEMBytes(), + sConf.ControlClientCert.SignedCertPEMBytes(), + sConf.ControlClientCert.PrivateKeyPEMBytes(), + ) } select {} }, } +func extractHost(s string) string { + i := strings.LastIndex(s, ":") + s = s[:i] + l := len(s) - 1 + if s[0] == '[' && s[l] == ']' { + s = s[1:l] + } + return s +} + func init() { ServerCmd.Flags().String("bind", "0.0.0.0:8080", "IP address and port to bind the user interface to") ServerCmd.Flags().String("control-bind", "localhost:8081", "IP address and port to bind the control interface to") ServerCmd.Flags().String("data-bind", "localhost:8082", "IP address and port to bind the data interface to") + ServerCmd.Flags().String("control-reachable-addr", "localhost:8081", "address (incl. port) through which the control interface can be reached (from the client's point of view)") + ServerCmd.Flags().String("certfile", "", "Cert file to serve thru https") ServerCmd.Flags().String("keyfile", "", "Key file to serve thru https") @@ -100,10 +148,19 @@ func validateServerCommandArguments(cmd *cobra.Command, args []string) error { return nil } -func Run(path string, debug bool) { +func Run( + path string, + debug bool, + controlServerCertPEM, + controlClientCertPEM, + controlClientCertPrivKeyPEM []byte, +) { logger.L.Printf("Running init program %+q", path) cmd := BuildCmd(path) cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("KAPOW_CONTROL_SERVER_CERT=%s", controlServerCertPEM)) + cmd.Env = append(cmd.Env, fmt.Sprintf("KAPOW_CONTROL_CLIENT_CERT=%s", controlClientCertPEM)) + cmd.Env = append(cmd.Env, fmt.Sprintf("KAPOW_CONTROL_CLIENT_KEY=%s", controlClientCertPrivKeyPEM)) var wg sync.WaitGroup if debug { diff --git a/internal/http/request.go b/internal/http/request.go index aa4b121d..f38010eb 100644 --- a/internal/http/request.go +++ b/internal/http/request.go @@ -17,30 +17,41 @@ package http import ( + "crypto/tls" + "crypto/x509" "errors" "io" "io/ioutil" "net/http" + "os" + + "github.com/BBVA/kapow/internal/logger" ) +var ControlClientGenerator = GenControlHTTPSClient + +func AsJSON(req *http.Request) { + req.Header.Add("Content-Type", "application/json") +} + // Get perform a request using Request with the GET method -func Get(url string, contentType string, r io.Reader, w io.Writer) error { - return Request("GET", url, contentType, r, w) +func Get(url string, r io.Reader, w io.Writer, clientGenerator func() *http.Client, reqTuner ...func(*http.Request)) error { + return Request("GET", url, r, w, clientGenerator, reqTuner...) } // Post perform a request using Request with the POST method -func Post(url string, contentType string, r io.Reader, w io.Writer) error { - return Request("POST", url, contentType, r, w) +func Post(url string, r io.Reader, w io.Writer, clientGenerator func() *http.Client, reqTuner ...func(*http.Request)) error { + return Request("POST", url, r, w, clientGenerator, reqTuner...) } // Put perform a request using Request with the PUT method -func Put(url string, contentType string, r io.Reader, w io.Writer) error { - return Request("PUT", url, contentType, r, w) +func Put(url string, r io.Reader, w io.Writer, clientGenerator func() *http.Client, reqTuner ...func(*http.Request)) error { + return Request("PUT", url, r, w, clientGenerator, reqTuner...) } // Delete perform a request using Request with the DELETE method -func Delete(url string, contentType string, r io.Reader, w io.Writer) error { - return Request("DELETE", url, contentType, r, w) +func Delete(url string, r io.Reader, w io.Writer, clientGenerator func() *http.Client, reqTuner ...func(*http.Request)) error { + return Request("DELETE", url, r, w, clientGenerator, reqTuner...) } var devnull = ioutil.Discard @@ -49,17 +60,24 @@ var devnull = ioutil.Discard // content of the given reader as the body and writing all the contents // of the response to the given writer. The reader and writer are // optional. -func Request(method string, url string, contentType string, r io.Reader, w io.Writer) error { +func Request(method string, url string, r io.Reader, w io.Writer, clientGenerator func() *http.Client, reqTuners ...func(*http.Request)) error { req, err := http.NewRequest(method, url, r) if err != nil { return err } - if contentType != "" { - req.Header.Add("Content-Type", contentType) + for _, reqTuner := range reqTuners { + reqTuner(req) } - res, err := new(http.Client).Do(req) + var client *http.Client + if clientGenerator == nil { + client = new(http.Client) + } else { + client = clientGenerator() + } + + res, err := client.Do(req) if err != nil { return err } @@ -81,3 +99,43 @@ func Request(method string, url string, contentType string, r io.Reader, w io.Wr return err } + +func GenControlHTTPSClient() *http.Client { + + serverCert, exists := os.LookupEnv("KAPOW_CONTROL_SERVER_CERT") + if !exists { + logger.L.Fatal("KAPOW_CONTROL_SERVER_CERT not in the environment") + } + + clientCert, exists := os.LookupEnv("KAPOW_CONTROL_CLIENT_CERT") + if !exists { + logger.L.Fatal("KAPOW_CONTROL_CLIENT_CERT not in the environment") + } + + clientKey, exists := os.LookupEnv("KAPOW_CONTROL_CLIENT_KEY") + if !exists { + logger.L.Fatal("KAPOW_CONTROL_CLIENT_KEY not in the environment") + } + + // Load client cert + clientTLSCert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientKey)) + if err != nil { + logger.L.Fatal(err) + } + + // Load Server cert + serverCertPool := x509.NewCertPool() + serverCertPool.AppendCertsFromPEM([]byte(serverCert)) + + // Setup HTTPS client + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{clientTLSCert}, + RootCAs: serverCertPool, + } + tlsConfig.BuildNameToCertificate() + transport := &http.Transport{TLSClientConfig: tlsConfig} + client := &http.Client{Transport: transport} + + // The client is always right! + return client +} diff --git a/internal/http/request_test.go b/internal/http/request_test.go index bd13c2c5..67198c54 100644 --- a/internal/http/request_test.go +++ b/internal/http/request_test.go @@ -29,7 +29,7 @@ func TestReturnErrorOnInvalidURL(t *testing.T) { defer gock.Off() gock.New("").Reply(200) - err := Request("GET", "://", "", nil, nil) + err := Request("GET", "://", nil, nil, nil) if err == nil { t.Errorf("Expected error not returned") } @@ -45,7 +45,7 @@ func TestRequestGivenMethod(t *testing.T) { mock.Method = "FOO" mock.Reply(200) - err := Request("FOO", "http://localhost", "", nil, nil) + err := Request("FOO", "http://localhost", nil, nil, nil) if err != nil { t.Errorf("Unexpected error on request") } @@ -60,7 +60,7 @@ func TestReturnHTTPErrorAsIs(t *testing.T) { customError := errors.New("FOO") gock.New("http://localhost").ReplyError(customError) - err := Request("GET", "http://localhost", "", nil, nil) + err := Request("GET", "http://localhost", nil, nil, nil) if errors.Unwrap(err) != customError { t.Errorf("Returned error is not the expected error: '%v'", err) } @@ -76,7 +76,7 @@ func TestReturnHTTPReasonAsErrorWhenUnsuccessful(t *testing.T) { Reply(http.StatusTeapot). BodyString(`{"reason": "I'm a teapot"}`) - err := Request("GET", "http://localhost", "", nil, nil) + err := Request("GET", "http://localhost", nil, nil, nil) if err == nil || err.Error() != http.StatusText(http.StatusTeapot) { t.Errorf("Reason should be returned as an error") } @@ -93,7 +93,7 @@ func TestCopyResponseBodyToWriter(t *testing.T) { rw := new(bytes.Buffer) - err := Request("GET", "http://localhost", "", nil, rw) + err := Request("GET", "http://localhost", nil, rw, nil) if err != nil { t.Errorf("Unexpected error %v", err) } @@ -119,7 +119,7 @@ func TestWriteToDevNullWhenNoWriter(t *testing.T) { defer func() { devnull = original }() - err := Request("GET", "http://localhost", "", nil, nil) + err := Request("GET", "http://localhost", nil, nil, nil) if err != nil { t.Errorf("Unexpected error %v", err) } @@ -135,14 +135,13 @@ func TestWriteToDevNullWhenNoWriter(t *testing.T) { } } -func TestSendContentType(t *testing.T) { +func TestSendContentTypeJSON(t *testing.T) { defer gock.Off() gock.New("http://localhost"). - MatchHeader("Content-Type", "foo/bar"). - HeaderPresent("Content-Type"). + MatchHeader("Content-Type", "application/json"). Reply(http.StatusOK) - err := Request("GET", "http://localhost", "foo/bar", nil, nil) + err := Request("GET", "http://localhost", nil, nil, nil, AsJSON) if err != nil { t.Errorf("Unexpected error '%v'", err.Error()) } @@ -158,7 +157,7 @@ func TestGetRequestsWithMethodGet(t *testing.T) { Get("/"). Reply(http.StatusOK) - err := Get("http://localhost/", "", nil, nil) + err := Get("http://localhost/", nil, nil, nil) if err != nil { t.Errorf("Unexpected error %q", err) @@ -175,7 +174,7 @@ func TestPostRequestsWithMethodPost(t *testing.T) { Post("/"). Reply(http.StatusOK) - err := Post("http://localhost/", "", nil, nil) + err := Post("http://localhost/", nil, nil, nil) if err != nil { t.Errorf("Unexpected error %q", err) @@ -192,7 +191,7 @@ func TestPutRequestsWithMethodPut(t *testing.T) { Put("/"). Reply(http.StatusOK) - err := Put("http://localhost/", "", nil, nil) + err := Put("http://localhost/", nil, nil, nil) if err != nil { t.Errorf("Unexpected error %q", err) @@ -209,7 +208,7 @@ func TestDeleteRequestsWithMethodDelete(t *testing.T) { Delete("/"). Reply(http.StatusOK) - err := Delete("http://localhost/", "", nil, nil) + err := Delete("http://localhost/", nil, nil, nil) if err != nil { t.Errorf("Unexpected error %q", err) diff --git a/internal/server/control/control_test.go b/internal/server/control/control_test.go index 52747e18..d93500cd 100644 --- a/internal/server/control/control_test.go +++ b/internal/server/control/control_test.go @@ -58,56 +58,6 @@ func checkErrorResponse(r *http.Response, expectedErrcode int, expectedReason st return errList } -func TestConfigRouterHasRoutesWellConfigured(t *testing.T) { - testCases := []struct { - pattern, method string - handler uintptr - mustMatch bool - vars []string - }{ - {"/routes/FOO", http.MethodGet, reflect.ValueOf(getRoute).Pointer(), true, []string{"id"}}, - {"/routes/FOO", http.MethodPut, reflect.ValueOf(defMethodNotAllowedHandler).Pointer(), true, []string{}}, - {"/routes/FOO", http.MethodPost, reflect.ValueOf(defMethodNotAllowedHandler).Pointer(), true, []string{}}, - {"/routes/FOO", http.MethodDelete, reflect.ValueOf(removeRoute).Pointer(), true, []string{"id"}}, - {"/routes", http.MethodGet, reflect.ValueOf(listRoutes).Pointer(), true, []string{}}, - {"/routes", http.MethodPut, reflect.ValueOf(defMethodNotAllowedHandler).Pointer(), true, []string{}}, - {"/routes", http.MethodPost, reflect.ValueOf(addRoute).Pointer(), true, []string{}}, - {"/routes", http.MethodDelete, reflect.ValueOf(defMethodNotAllowedHandler).Pointer(), true, []string{}}, - {"/", http.MethodGet, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/", http.MethodPut, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/", http.MethodPost, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/", http.MethodDelete, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/FOO", http.MethodGet, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/FOO", http.MethodPut, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/FOO", http.MethodPost, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - {"/FOO", http.MethodDelete, reflect.ValueOf(defNotFoundHandler).Pointer(), true, []string{}}, - } - r := configRouter() - - for _, tc := range testCases { - rm := mux.RouteMatch{} - rq, _ := http.NewRequest(tc.method, tc.pattern, nil) - if matched := r.Match(rq, &rm); tc.mustMatch == matched { - if tc.mustMatch { - // Check for Handler match. - realHandler := reflect.ValueOf(rm.Handler).Pointer() - if realHandler != tc.handler { - t.Errorf("Handler mismatch. Expected: %X, got: %X", tc.handler, realHandler) - } - - // Check for variables - for _, vn := range tc.vars { - if _, exists := rm.Vars[vn]; !exists { - t.Errorf("Variable not present: %s", vn) - } - } - } - } else { - t.Errorf("Route mismatch: %+v", tc) - } - } -} - func TestPathValidatorNoErrorWhenCorrectPath(t *testing.T) { err := pathValidator("/routes/{routeID}") diff --git a/internal/server/control/server.go b/internal/server/control/server.go index 82ce965b..c4ba626f 100644 --- a/internal/server/control/server.go +++ b/internal/server/control/server.go @@ -17,24 +17,47 @@ package control import ( + "crypto/tls" + "crypto/x509" "net" "net/http" "sync" + "github.com/BBVA/kapow/internal/certs" "github.com/BBVA/kapow/internal/logger" ) // Run Starts the control server listening in bindAddr -func Run(bindAddr string, wg *sync.WaitGroup) { +func Run(bindAddr string, wg *sync.WaitGroup, serverCert, clientCert certs.Cert) { - listener, err := net.Listen("tcp", bindAddr) + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(clientCert.SignedCertPEMBytes()) + + ln, err := net.Listen("tcp", bindAddr) if err != nil { logger.L.Fatal(err) } + server := &http.Server{ + Addr: bindAddr, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{ + tls.Certificate{ + Certificate: [][]byte{serverCert.SignedCert}, + PrivateKey: serverCert.PrivKey, + Leaf: serverCert.X509Cert, + }, + }, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caCertPool, + }, + Handler: configRouter(), + } + // Signal startup logger.L.Printf("ControlServer listening at %s\n", bindAddr) wg.Done() - logger.L.Fatal(http.Serve(listener, configRouter())) + // Listen to HTTPS connections with the server certificate and wait + logger.L.Fatal(server.ServeTLS(ln, "", "")) } diff --git a/internal/server/server.go b/internal/server/server.go index ea0d6caa..08a2c2a1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -19,6 +19,7 @@ package server import ( "sync" + "github.com/BBVA/kapow/internal/certs" "github.com/BBVA/kapow/internal/server/control" "github.com/BBVA/kapow/internal/server/data" "github.com/BBVA/kapow/internal/server/user" @@ -34,13 +35,16 @@ type ServerConfig struct { ClientAuth, Debug bool + + ControlServerCert certs.Cert + ControlClientCert certs.Cert } // StartServer Starts one instance of each server in a goroutine and remains listening on a channel for trace events generated by them func StartServer(config ServerConfig) { var wg = sync.WaitGroup{} wg.Add(3) - go control.Run(config.ControlBindAddr, &wg) + go control.Run(config.ControlBindAddr, &wg, config.ControlServerCert, config.ControlClientCert) go data.Run(config.DataBindAddr, &wg) go user.Run(config.UserBindAddr, &wg, config.CertFile, config.KeyFile, config.ClientCaFile, config.ClientAuth, config.Debug) diff --git a/spec/README.md b/spec/README.md index bd16e4e3..9a868ba0 100644 --- a/spec/README.md +++ b/spec/README.md @@ -130,6 +130,7 @@ whole lifetime of the server. * Kapow! implementations should follow a general principle of robustness: be conservative in what you do, be liberal in what you accept from others. * We reuse conventions of well-established software projects, such as Docker. +* Secure by default, the Control API can *only* be accessed using mTLS. * All requests and responses will leverage JSON as the data encoding method. * The API calls responses have several parts: * The HTTP status code (e.g., `400`, which is a bad request). The target @@ -178,6 +179,30 @@ Content-Length: 25 ``` +## mTLS + +The Kapow! server generates a pair of keys and certificates, one for the +server, the other for the configuring client. The necessary elements will be +communicated to the client (the init program) via a set of environment +variables. + +The aforementioned variables are named: + +- `KAPOW_CONTROL_SERVER_CERT`: server certificate. +- `KAPOW_CONTROL_CLIENT_CERT`: client certificate. +- `KAPOW_CONTROL_CLIENT_KEY`: client private key. + +Note that all variables contain x509 PEM-encoded values. +Also note that the server private key is not communicated in any way. + +Following the mTLS discipline, the client must ensure upon connecting to the +server that its certificate matches the one stored in +`KAPOW_CONTROL_SERVER_CERT`. + +Conversely, the server must only communicate with clients whose certificate +matches the one stored in `KAPOW_CONTROL_CLIENT_CERT`. + + ## API Elements Kapow! provides a way to control its internal state through these elements. @@ -606,8 +631,6 @@ Commands: ``` -### `kapow server` - This command runs the Kapow! server, which is the core of Kapow!. If run without parameters, it will run an unconfigured server. It can accept a path to an executable file, the init program, which can be a shell script that @@ -615,7 +638,7 @@ contains commands to configure the *Kapow!* server. The init program can leverage the `kapow route` command, which is used to define a route. The `kapow route` command needs a way to reach the *Kapow!* server, -and for that, `kapow` provides the `KAPOW_DATA_URL` variable in the environment +and for that, `kapow` provides the `KAPOW_CONTROL_URL` variable in the environment of the aforementioned init program. Every time the *Kapow!* server receives a request, it will spawn a process to @@ -655,7 +678,10 @@ To deregister a route you must provide a *route_id*. #### **Environment** -- `KAPOW_DATA_URL` +- `KAPOW_CONTROL_URL` +- `KAPOW_CONTROL_SERVER_CERT` +- `KAPOW_CONTROL_CLIENT_CERT` +- `KAPOW_CONTROL_CLIENT_KEY` #### **Help** @@ -696,7 +722,7 @@ Options: $ kapow route add -X GET '/list/{ip}' -c 'nmap -sL $(kapow get /request/matches/ip) | kapow set /response/body' ``` -### `request` +### `kapow get` Exposes the requests' resources. @@ -713,7 +739,7 @@ $ kapow get /request/body ``` -### `response` +### `kapow set` Exposes the response's resources. diff --git a/spec/test/.envrc b/spec/test/.envrc new file mode 100644 index 00000000..4a4726a5 --- /dev/null +++ b/spec/test/.envrc @@ -0,0 +1 @@ +use_nix diff --git a/spec/test/Dockerfile b/spec/test/Dockerfile index bd8290d7..f36ff998 100644 --- a/spec/test/Dockerfile +++ b/spec/test/Dockerfile @@ -1,16 +1,18 @@ -FROM python:3.7-alpine +FROM nixos/nix:2.3.6 # Install CircleCI requirements for base images # https://circleci.com/docs/2.0/custom-images/ -RUN apk upgrade --update-cache \ - && apk add git openssh-server tar gzip ca-certificates +# RUN apk upgrade --update-cache \ +# && apk add git openssh-server tar gzip ca-certificates # Install Kapow! Spec Test Suite RUN mkdir -p /usr/src/ksts WORKDIR /usr/src/ksts COPY features /usr/src/ksts/features -COPY Pipfile Pipfile.lock /usr/src/ksts/ -RUN pip install --upgrade pip \ - && pip install pipenv \ - && pipenv install --deploy --system \ - && rm -f Pipfile Pipfile.lock +# COPY Pipfile Pipfile.lock /usr/src/ksts/ +# RUN pip install --upgrade pip \ +# && pip install pipenv \ +# && pipenv install --deploy --system \ +# && rm -f Pipfile Pipfile.lock +COPY ./*.nix ./ +ENTRYPOINT [ "nix-shell", "--command" ] diff --git a/spec/test/Makefile b/spec/test/Makefile index b11f077a..6033104c 100644 --- a/spec/test/Makefile +++ b/spec/test/Makefile @@ -1,23 +1,20 @@ -.PHONY: lint wip test fix catalog sync +.PHONY: all lint wip test fix catalog -all: checkbin sync test +all: checkbin test -sync: - pipenv sync lint: gherkin-lint wip: - KAPOW_DEBUG_TESTS=1 pipenv run behave --stop --wip + KAPOW_DEBUG_TESTS=1 behave --stop --wip -k test: lint - pipenv run behave --no-capture --tags=~@skip + behave --no-capture --tags=~@skip fix: lint - KAPOW_DEBUG_TESTS=1 pipenv run behave --stop --no-capture --tags=~@skip + KAPOW_DEBUG_TESTS=1 behave --stop --no-capture --tags=~@skip catalog: - pipenv run behave --format steps.usage --dry-run --no-summary -q -clean: - pipenv --rm + behave --format steps.usage --dry-run --no-summary -q checkbin: @which kapow >/dev/null || (echo "ERROR: Your kapow binary is not present in PATH" && exit 1) -testpoc: sync - pipenv run pip install -r ../../testutils/poc/requirements.txt - PATH=../../testutils/poc:$$PATH KAPOW_CONTROL_URL=http://localhost:8081 KAPOW_DATA_URL=http://localhost:8081 pipenv run behave --no-capture --tags=~@skip +testpoc: + PATH=../../testutils/poc:$$PATH behave --no-capture --tags=~@skip +wippoc: + PATH=../../testutils/poc:$$PATH behave --no-capture --tags=@wip -k diff --git a/spec/test/features/control/list/success.feature b/spec/test/features/control/list/success.feature index 0361043a..769f8422 100644 --- a/spec/test/features/control/list/success.feature +++ b/spec/test/features/control/list/success.feature @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +@server Feature: Listing routes in a Kapow! server. Listing routes allows users to know what URLs are available on a Kapow! server. The List endpoint returns diff --git a/spec/test/features/control/mtls.feature b/spec/test/features/control/mtls.feature new file mode 100644 index 00000000..3b6a27dd --- /dev/null +++ b/spec/test/features/control/mtls.feature @@ -0,0 +1,95 @@ +# +# Copyright 2021 Banco Bilbao Vizcaya Argentaria, S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Feature: Communications with the control interface are secured with mTLS. + Trust is anchored via certificate pinning. + The Kapow! server only allows connections from trusted clients. + The Kapow! clients only establish connections to trusted servers. + + @server + Scenario: Reject clients not providing a certificate. + + Given I have a running Kapow! server + When I try to connect to the control API without providing a certificate + Then I get a connection error + + @server + Scenario: Reject clients providing an invalid certificate. + + Given I have a running Kapow! server + When I try to connect to the control API providing an invalid certificate + Then I get a connection error + + @client + Scenario: Connect to servers providing a valid certificate. + A valid certificate is the one provided via envvars. + + Given a test HTTPS server on the control port + When I run the following command + """ + $ kapow route list + """ + And the HTTPS server receives a "GET" request to "/routes" + And the server responds with + | field | value | + | status | 200 | + | headers.Content-Type | application/json | + | body | [] | + Then the command exits with "0" + + @client + Scenario: Reject servers providing an invalid certificate. + + Given a test HTTPS server on the control port + When I run the following command (with invalid certs) + """ + $ kapow route list + """ + Then the command exits immediately with "1" + + @server + Scenario Outline: The control server is accessible through an alternative address + The automatically generated certificated contains the Alternate Name + provided via the `--control-reachable-addr` parameter. + + Given I launch the server with the following extra arguments + """ + --control-reachable-addr "" + """ + When I inspect the automatically generated control server certificate + Then the extension "Subject Alternative Name" contains "" of type "" + + Examples: + | reachable_addr | value | type | + | localhost:8081 | localhost | DNSName | + | 127.0.0.1:8081 | 127.0.0.1 | IPAddress | + | foo.bar:8081 | foo.bar | DNSName | + | 4.2.2.4:8081 | 4.2.2.4 | IPAddress | + | [2600::]:8081 | 2600:: | IPAddress | + + + @e2e + Scenario: Control server dialog using mTLS + If the user provides the corresponding certificates to the + `kapow route` subcommand, the communication should be possible. + + Given I have a just started Kapow! server + When I run the following command (setting the control certs environment variables) + """ + $ kapow route list + + """ + Then the command exits with "0" diff --git a/spec/test/features/environment.py b/spec/test/features/environment.py index 1dffa4c2..52a05f4c 100644 --- a/spec/test/features/environment.py +++ b/spec/test/features/environment.py @@ -15,25 +15,43 @@ # import tempfile import os +import signal +from contextlib import suppress - -def before_scenario(context, scenario): - # Create the request_handler FIFO +def tmpfifo(): while True: - context.handler_fifo_path = tempfile.mktemp() # Safe because using - # mkfifo + fifo_path = tempfile.mktemp() # The usage mkfifo make this safe try: - os.mkfifo(context.handler_fifo_path) + os.mkfifo(fifo_path) except OSError: # The file already exist pass else: break + return fifo_path + + +def before_scenario(context, scenario): + context.handler_fifo_path = tmpfifo() + context.init_script_fifo_path = tmpfifo() + def after_scenario(context, scenario): + # Real Kapow! server being tested if hasattr(context, 'server'): context.server.terminate() context.server.wait() os.unlink(context.handler_fifo_path) + os.unlink(context.init_script_fifo_path) + + # Mock HTTP server for testing + if hasattr(context, 'httpserver'): + context.response_ready.set() + context.httpserver.shutdown() + context.httpserver_thread.join() + + if getattr(context, 'testing_handler_pid', None) is not None: + with suppress(ProcessLookupError): + os.kill(int(context.testing_handler_pid), signal.SIGTERM) diff --git a/spec/test/features/steps/get_environment.py b/spec/test/features/steps/get_environment.py new file mode 100755 index 00000000..bb6e09cc --- /dev/null +++ b/spec/test/features/steps/get_environment.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import json +import os +import sys + +if __name__ == '__main__': + with open(os.environ['SPECTEST_FIFO'], 'w') as fifo: + json.dump(dict(os.environ), fifo) diff --git a/spec/test/features/steps/steps.py b/spec/test/features/steps/steps.py index f16468aa..21ea795f 100644 --- a/spec/test/features/steps/steps.py +++ b/spec/test/features/steps/steps.py @@ -13,26 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from contextlib import suppress +from contextlib import suppress, contextmanager +from multiprocessing.pool import ThreadPool from time import sleep +import datetime +import http.server +import ipaddress import json +import logging import os import shlex import signal import socket +import ssl import subprocess import sys import tempfile import threading -from multiprocessing.pool import ThreadPool import time -import requests -from environconfig import EnvironConfig, StringVar, IntVar, BooleanVar from comparedict import is_subset +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import serialization +from cryptography import x509 +from cryptography.x509.oid import NameOID, ExtensionOID +from environconfig import EnvironConfig, StringVar, IntVar, BooleanVar +from requests import exceptions as requests_exceptions import jsonexample - -import logging +import requests WORD2POS = {"first": 0, "second": 1, "last": -1} @@ -44,7 +54,8 @@ class Env(EnvironConfig): KAPOW_SERVER_CMD = StringVar(default="kapow server") #: Where the Control API is - KAPOW_CONTROL_URL = StringVar(default="http://localhost:8081") + KAPOW_CONTROL_URL = StringVar(default="https://localhost:8081") + KAPOW_CONTROL_PORT = IntVar(default=8081) #: Where the Data API is KAPOW_DATA_URL = StringVar(default="http://localhost:8082") @@ -52,7 +63,9 @@ class Env(EnvironConfig): #: Where the User Interface is KAPOW_USER_URL = StringVar(default="http://localhost:8080") - KAPOW_BOOT_TIMEOUT = IntVar(default=1000) + KAPOW_CONTROL_TOKEN = StringVar(default="TEST-SPEC-CONTROL-TOKEN") + + KAPOW_BOOT_TIMEOUT = IntVar(default=3000) KAPOW_DEBUG_TESTS = BooleanVar(default=False) @@ -77,37 +90,134 @@ class Env(EnvironConfig): requests_log.setLevel(logging.DEBUG) requests_log.propagate = True -def run_kapow_server(context): + +def generate_ssl_cert(subject_name, alternate_name): + # Generate our key + key = rsa.generate_private_key( + public_exponent=65537, + key_size=4096, + ) + # Various details about who we are. For a self-signed certificate the + # subject and issuer are always the same. + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, subject_name), + ]) + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + # Our certificate will be valid for 10 days + datetime.datetime.utcnow() + datetime.timedelta(days=10) + ).add_extension( + x509.SubjectAlternativeName([x509.DNSName(alternate_name)]), + critical=True, + ).add_extension( + x509.ExtendedKeyUsage( + [x509.oid.ExtendedKeyUsageOID.SERVER_AUTH + if subject_name.endswith('_server') + else x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), + critical=True, + # Sign our certificate with our private key + ).sign(key, hashes.SHA256()) + + key_bytes = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + crt_bytes = cert.public_bytes(serialization.Encoding.PEM) + + return (key_bytes, crt_bytes) + + +@contextmanager +def mtls_client(context): + with tempfile.NamedTemporaryFile(suffix='.crt', encoding='utf-8', mode='w') as srv_cert, \ + tempfile.NamedTemporaryFile(suffix='.crt', encoding='utf-8', mode='w') as cli_cert, \ + tempfile.NamedTemporaryFile(suffix='.key', encoding='utf-8', mode='w') as cli_key: + srv_cert.write(context.init_script_environ["KAPOW_CONTROL_SERVER_CERT"]) + srv_cert.file.flush() + cli_cert.write(context.init_script_environ["KAPOW_CONTROL_CLIENT_CERT"]) + cli_cert.file.flush() + cli_key.write(context.init_script_environ["KAPOW_CONTROL_CLIENT_KEY"]) + cli_key.file.flush() + session=requests.Session() + session.verify=srv_cert.name + session.cert=(cli_cert.name, cli_key.name) + yield session + + +def is_port_open(port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + return sock.connect_ex(('127.0.0.1', port)) == 0 + + +def run_kapow_server(context, extra_args=""): + assert (not is_port_open(Env.KAPOW_CONTROL_PORT)), "Another process is already bound" + context.server = subprocess.Popen( - shlex.split(Env.KAPOW_SERVER_CMD), + shlex.split(Env.KAPOW_SERVER_CMD) + shlex.split(extra_args) + [os.path.join(HERE, "get_environment.py")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={'SPECTEST_FIFO': context.init_script_fifo_path, **os.environ}, shell=False) # Check process is running with reachable APIs open_ports = False for _ in range(Env.KAPOW_BOOT_TIMEOUT): - is_running = context.server.poll() is None - assert is_running, "Server is not running!" - with suppress(requests.exceptions.ConnectionError): - open_ports = ( - requests.head(Env.KAPOW_CONTROL_URL, timeout=1).status_code - and requests.head(Env.KAPOW_DATA_URL, timeout=1).status_code) - if open_ports: + with suppress(requests_exceptions.ConnectionError): + if is_port_open(Env.KAPOW_CONTROL_PORT): + open_ports = True break sleep(.01) assert open_ports, "API is unreachable after KAPOW_BOOT_TIMEOUT" + # Get init_script enviroment via fifo + with open(context.init_script_fifo_path, 'r') as fifo: + context.init_script_environ = json.load(fifo) + + @given('I have a just started Kapow! server') @given('I have a running Kapow! server') def step_impl(context): run_kapow_server(context) +@given(u'I launch the server with the following extra arguments') +def step_impl(context): + run_kapow_server(context, context.text) + + +@when('I request a route listing without providing a Control Access Token') +def step_impl(context): + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + + +@when('I request a route listing without providing an empty Control Access Token') +def step_impl(context): + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + + +@when(u'I request a route listing providing a bad Control Access Token') +def step_impl(context): + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + + @when('I request a routes listing') def step_impl(context): - context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + with mtls_client(context) as requests: + context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") @given('I have a Kapow! server with the following routes') @@ -117,10 +227,12 @@ def step_impl(context): if not hasattr(context, 'table'): raise RuntimeError("A table must be set for this step.") - for row in context.table: - response = requests.post(f"{Env.KAPOW_CONTROL_URL}/routes", - json={h: row[h] for h in row.headings}) - response.raise_for_status() + with mtls_client(context) as requests: + for row in context.table: + response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + json={h: row[h] for h in row.headings}) + response.raise_for_status() @given('I have a Kapow! server with the following testing routes') @@ -130,15 +242,16 @@ def step_impl(context): if not hasattr(context, 'table'): raise RuntimeError("A table must be set for this step.") - for row in context.table: - response = requests.post( - f"{Env.KAPOW_CONTROL_URL}/routes", - json={"entrypoint": " ".join( - [sys.executable, - shlex.quote(os.path.join(HERE, "testinghandler.py")), - shlex.quote(context.handler_fifo_path)]), # Created in before_scenario - **{h: row[h] for h in row.headings}}) - response.raise_for_status() + with mtls_client(context) as requests: + for row in context.table: + response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + json={"entrypoint": " ".join( + [sys.executable, + shlex.quote(os.path.join(HERE, "testinghandler.py")), + shlex.quote(context.handler_fifo_path)]), # Created in before_scenario + **{h: row[h] for h in row.headings}}) + response.raise_for_status() def testing_request(context, request_fn): # Run the request in background @@ -165,15 +278,17 @@ def _request(): @when('I release the testing request') def step_impl(context): os.kill(int(context.testing_handler_pid), signal.SIGTERM) + context.testing_handler_pid = None context.testing_response = context.testing_request.get() @when('I append the route') def step_impl(context): - context.response = requests.post(f"{Env.KAPOW_CONTROL_URL}/routes", - data=context.text, - headers={"Content-Type": "application/json"}) - + with mtls_client(context) as requests: + context.response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + data=context.text, + headers={"Content-Type": "application/json"}) @then('I get {code} as response code') def step_impl(context, code): @@ -212,50 +327,62 @@ def step_impl(context): @when('I delete the route with id "{id}"') def step_impl(context, id): - context.response = requests.delete(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + context.response = requests.delete( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I insert the route') def step_impl(context): - context.response = requests.put(f"{Env.KAPOW_CONTROL_URL}/routes", - headers={"Content-Type": "application/json"}, - data=context.text) + with mtls_client(context) as requests: + context.response = requests.put( + f"{Env.KAPOW_CONTROL_URL}/routes", + headers={"Content-Type": "application/json"}, + data=context.text) @when('I try to append with this malformed JSON document') def step_impl(context): - context.response = requests.post( - f"{Env.KAPOW_CONTROL_URL}/routes", - headers={"Content-Type": "application/json"}, - data=context.text) + with mtls_client(context) as requests: + context.response = requests.post( + f"{Env.KAPOW_CONTROL_URL}/routes", + headers={"Content-Type": "application/json"}, + data=context.text) @when('I delete the {order} route') def step_impl(context, order): - idx = WORD2POS.get(order) - routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") - id = routes.json()[idx]["id"] - context.response = requests.delete(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + idx = WORD2POS.get(order) + routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + id = routes.json()[idx]["id"] + context.response = requests.delete( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I try to insert with this JSON document') def step_impl(context): - context.response = requests.put( - f"{Env.KAPOW_CONTROL_URL}/routes", - headers={"Content-Type": "application/json"}, - data=context.text) + with mtls_client(context) as requests: + context.response = requests.put( + f"{Env.KAPOW_CONTROL_URL}/routes", + headers={"Content-Type": "application/json"}, + data=context.text) @when('I get the route with id "{id}"') def step_impl(context, id): - context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + context.response = requests.get( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I get the {order} route') def step_impl(context, order): - idx = WORD2POS.get(order) - routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") - id = routes.json()[idx]["id"] - context.response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes/{id}") + with mtls_client(context) as requests: + idx = WORD2POS.get(order) + routes = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes") + id = routes.json()[idx]["id"] + context.response = requests.get( + f"{Env.KAPOW_CONTROL_URL}/routes/{id}") @when('I get the resource "{resource}"') @@ -316,3 +443,216 @@ def step_impl(context, value, fieldType, elementName): raise ValueError("Unknown fieldtype {fieldType!r}") assert actual == value, f"Expecting {fieldType} {elementName!r} to be {value!r}, got {actual!r} insted" + + +@given('a test HTTPS server on the {port} port') +def step_impl(context, port): + context.request_ready = threading.Event() + context.request_ready.clear() + context.response_ready = threading.Event() + context.response_ready.clear() + + class SaveResponseHandler(http.server.BaseHTTPRequestHandler): + def do_verb(self): + context.request_response = self + context.request_ready.set() + context.response_ready.wait() + do_GET=do_verb + do_POST=do_verb + do_PUT=do_verb + do_DELETE=do_verb + do_HEAD=do_verb + + if port == "control": + port = 8081 + elif port == "data": + port = 8082 + else: + raise ValueError(f"Unknown port {port}") + + context.httpserver = http.server.HTTPServer(('127.0.0.1', port), + SaveResponseHandler) + + context.srv_key, context.srv_crt = generate_ssl_cert("control_server", "localhost") + context.cli_key, context.cli_crt = generate_ssl_cert("control_client", "localhost") + with tempfile.NamedTemporaryFile(suffix=".key") as key_file, \ + tempfile.NamedTemporaryFile(suffix=".crt") as crt_file: + key_file.write(context.srv_key) + key_file.flush() + crt_file.write(context.srv_crt) + crt_file.flush() + context.httpserver.socket = ssl.wrap_socket( + context.httpserver.socket, + keyfile=key_file.name, + certfile=crt_file.name, + server_side=True) + context.httpserver_thread = threading.Thread( + target=context.httpserver.serve_forever, + daemon=True) + context.httpserver_thread.start() + + +def run_command_with_certs(context, srv_crt, cli_crt, cli_key): + _, command = context.text.split('$') + command = command.lstrip() + + def exec_in_thread(): + context.command = subprocess.Popen( + command, + shell=True, + env={'KAPOW_CONTROL_SERVER_CERT': srv_crt, + 'KAPOW_CONTROL_CLIENT_CERT': cli_crt, + 'KAPOW_CONTROL_CLIENT_KEY': cli_key, + **os.environ}) + context.command.wait() + + context.command_thread = threading.Thread(target=exec_in_thread, daemon=True) + context.command_thread.start() + +@step('I run the following command (with invalid certs)') +def step_impl(context): + invalid_srv_crt, _ = generate_ssl_cert("invalid_control_server", + "localhost") + run_command_with_certs(context, + invalid_srv_crt, + context.cli_crt, + context.cli_key) + + +@step('I run the following command') +def step_impl(context): + run_command_with_certs(context, + context.srv_crt, + context.cli_crt, + context.cli_key) + + +@when('I run the following command (setting the control certs environment variables)') +def step_impl(context): + run_command_with_certs( + context, + context.init_script_environ["KAPOW_CONTROL_SERVER_CERT"], + context.init_script_environ["KAPOW_CONTROL_CLIENT_CERT"], + context.init_script_environ["KAPOW_CONTROL_CLIENT_KEY"]) + + +@step('the HTTPS server receives a "{method}" request to "{path}"') +def step_impl(context, method, path): + context.request_ready.wait() + assert context.request_response.command == method, f"Method {context.request_response.command} is not {method}" + assert context.request_response.path == path, f"Method {context.request_response.path} is not {path}" + + + +@then('the received request has the header "{name}" set to "{value}"') +def step_impl(context, name, value): + context.request_ready.wait() + matching = context.request_response.headers[name] + assert matching, f"Header {name} not found" + assert matching == value, f"Value of header doesn't match. {matching} != {value}" + + +@when('the server responds with') +def step_impl(context): + # TODO: set the fields given in the table + has_body = False + for row in context.table: + if row['field'] == 'status': + context.request_response.send_response(int(row['value'])) + elif row['field'].startswith('headers.'): + _, header = row['field'].split('.') + context.request_response.send_header(header, row['value']) + elif row['field'] == 'body': + has_body = True + payload = row['value'].encode('utf-8') + context.request_response.send_header('Content-Length', str(len(payload))) + context.request_response.end_headers() + context.request_response.wfile.write(payload) + + if not has_body: + context.request_response.send_header('Content-Length', '0') + context.request_response.end_headers() + + context.response_ready.set() + +@then('the command exits {immediately} with "{returncode}"') +@then('the command exits with "{returncode}"') +def step_impl(context, returncode, immediately=False): + context.command_thread.join(timeout=3.0 if immediately else None) + if context.command_thread.is_alive(): + try: + print("killing in the name of") + context.command.kill() + finally: + assert False, "The command is still alive" + + else: + context.command.wait() + assert context.command.returncode == int(returncode), f"Command returned {context.command.returncode} instead of {returncode}" + + +@then('the received request doesn\'t have the header "{name}" set') +def step_impl(context, name): + context.request_ready.wait() + assert name not in context.request_response.headers, f"Header {name} found" + + +@when('I try to connect to the control API without providing a certificate') +def step_impl(context): + try: + context.request_response = requests.get(f"{Env.KAPOW_CONTROL_URL}/routes", verify=False) + except Exception as exc: + context.request_response = exc + + +@then(u'I get a connection error') +def step_impl(context): + assert issubclass(type(context.request_response), Exception), context.request_response + + +@when(u'I try to connect to the control API providing an invalid certificate') +def step_impl(context): + key, cert = generate_ssl_cert("foo", "localhost") + with tempfile.NamedTemporaryFile(suffix='.crt') as cert_file, \ + tempfile.NamedTemporaryFile(suffix='.key') as key_file: + cert_file.write(cert) + cert_file.flush() + key_file.write(key) + key_file.flush() + with requests.Session() as session: + session.cert = (cert_file.name, key_file.name) + session.verify = False + try: + context.request_response = session.get( + f"{Env.KAPOW_CONTROL_URL}/routes") + except Exception as exc: + context.request_response = exc + + + +@when('I inspect the automatically generated control server certificate') +def step_impl(context): + context.control_server_cert = x509.load_pem_x509_certificate( + context.init_script_environ["KAPOW_CONTROL_SERVER_CERT"].encode('ascii')) + + +@then('the extension "{extension}" contains "{value}" of type "{typename}"') +def step_impl(context, extension, value, typename): + if extension == 'Subject Alternative Name': + oid = ExtensionOID.SUBJECT_ALTERNATIVE_NAME + else: + raise NotImplementedError(f'Unknown extension {extension}') + + if typename == 'DNSName': + type_ = x509.DNSName + converter = lambda x: x + elif typename == 'IPAddress': + type_ = x509.IPAddress + converter = ipaddress.ip_address + else: + raise NotImplementedError(f'Unknown type {typename}') + + ext = context.control_server_cert.extensions.get_extension_for_oid(oid) + values = ext.value.get_values_for_type(type_) + + assert converter(value) in values, f"Value {value} not in {values}" diff --git a/spec/test/node-dependencies.nix b/spec/test/node-dependencies.nix new file mode 100644 index 00000000..c970861a --- /dev/null +++ b/spec/test/node-dependencies.nix @@ -0,0 +1,17 @@ +# This file has been generated by node2nix 1.8.0. Do not edit! + +{pkgs ? import { + inherit system; + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}: + +let + nodeEnv = import ./node-env.nix { + inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile; + inherit nodejs; + libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; + }; +in +import ./node-packages.nix { + inherit (pkgs) fetchurl fetchgit; + inherit nodeEnv; +} \ No newline at end of file diff --git a/spec/test/node-env.nix b/spec/test/node-env.nix new file mode 100644 index 00000000..e1abf530 --- /dev/null +++ b/spec/test/node-env.nix @@ -0,0 +1,542 @@ +# This file originates from node2nix + +{stdenv, nodejs, python2, utillinux, libtool, runCommand, writeTextFile}: + +let + python = if nodejs ? python then nodejs.python else python2; + + # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise + tarWrapper = runCommand "tarWrapper" {} '' + mkdir -p $out/bin + + cat > $out/bin/tar <> $out/nix-support/hydra-build-products + ''; + }; + + includeDependencies = {dependencies}: + stdenv.lib.optionalString (dependencies != []) + (stdenv.lib.concatMapStrings (dependency: + '' + # Bundle the dependencies of the package + mkdir -p node_modules + cd node_modules + + # Only include dependencies if they don't exist. They may also be bundled in the package. + if [ ! -e "${dependency.name}" ] + then + ${composePackage dependency} + fi + + cd .. + '' + ) dependencies); + + # Recursively composes the dependencies of a package + composePackage = { name, packageName, src, dependencies ? [], ... }@args: + builtins.addErrorContext "while evaluating node package '${packageName}'" '' + DIR=$(pwd) + cd $TMPDIR + + unpackFile ${src} + + # Make the base dir in which the target dependency resides first + mkdir -p "$(dirname "$DIR/${packageName}")" + + if [ -f "${src}" ] + then + # Figure out what directory has been unpacked + packageDir="$(find . -maxdepth 1 -type d | tail -1)" + + # Restore write permissions to make building work + find "$packageDir" -type d -exec chmod u+x {} \; + chmod -R u+w "$packageDir" + + # Move the extracted tarball into the output folder + mv "$packageDir" "$DIR/${packageName}" + elif [ -d "${src}" ] + then + # Get a stripped name (without hash) of the source directory. + # On old nixpkgs it's already set internally. + if [ -z "$strippedName" ] + then + strippedName="$(stripHash ${src})" + fi + + # Restore write permissions to make building work + chmod -R u+w "$strippedName" + + # Move the extracted directory into the output folder + mv "$strippedName" "$DIR/${packageName}" + fi + + # Unset the stripped name to not confuse the next unpack step + unset strippedName + + # Include the dependencies of the package + cd "$DIR/${packageName}" + ${includeDependencies { inherit dependencies; }} + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + ''; + + pinpointDependencies = {dependencies, production}: + let + pinpointDependenciesFromPackageJSON = writeTextFile { + name = "pinpointDependencies.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function resolveDependencyVersion(location, name) { + if(location == process.env['NIX_STORE']) { + return null; + } else { + var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); + + if(fs.existsSync(dependencyPackageJSON)) { + var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); + + if(dependencyPackageObj.name == name) { + return dependencyPackageObj.version; + } + } else { + return resolveDependencyVersion(path.resolve(location, ".."), name); + } + } + } + + function replaceDependencies(dependencies) { + if(typeof dependencies == "object" && dependencies !== null) { + for(var dependency in dependencies) { + var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); + + if(resolvedVersion === null) { + process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); + } else { + dependencies[dependency] = resolvedVersion; + } + } + } + } + + /* Read the package.json configuration */ + var packageObj = JSON.parse(fs.readFileSync('./package.json')); + + /* Pinpoint all dependencies */ + replaceDependencies(packageObj.dependencies); + if(process.argv[2] == "development") { + replaceDependencies(packageObj.devDependencies); + } + replaceDependencies(packageObj.optionalDependencies); + + /* Write the fixed package.json file */ + fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); + ''; + }; + in + '' + node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} + + ${stdenv.lib.optionalString (dependencies != []) + '' + if [ -d node_modules ] + then + cd node_modules + ${stdenv.lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} + cd .. + fi + ''} + ''; + + # Recursively traverses all dependencies of a package and pinpoints all + # dependencies in the package.json file to the versions that are actually + # being used. + + pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: + '' + if [ -d "${packageName}" ] + then + cd "${packageName}" + ${pinpointDependencies { inherit dependencies production; }} + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + fi + ''; + + # Extract the Node.js source code which is used to compile packages with + # native bindings + nodeSources = runCommand "node-sources" {} '' + tar --no-same-owner --no-same-permissions -xf ${nodejs.src} + mv node-* $out + ''; + + # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) + addIntegrityFieldsScript = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function augmentDependencies(baseDir, dependencies) { + for(var dependencyName in dependencies) { + var dependency = dependencies[dependencyName]; + + // Open package.json and augment metadata fields + var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); + var packageJSONPath = path.join(packageJSONDir, "package.json"); + + if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored + console.log("Adding metadata fields to: "+packageJSONPath); + var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); + + if(dependency.integrity) { + packageObj["_integrity"] = dependency.integrity; + } else { + packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. + } + + if(dependency.resolved) { + packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided + } else { + packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. + } + + if(dependency.from !== undefined) { // Adopt from property if one has been provided + packageObj["_from"] = dependency.from; + } + + fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); + } + + // Augment transitive dependencies + if(dependency.dependencies !== undefined) { + augmentDependencies(packageJSONDir, dependency.dependencies); + } + } + } + + if(fs.existsSync("./package-lock.json")) { + var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); + + if(packageLock.lockfileVersion !== 1) { + process.stderr.write("Sorry, I only understand lock file version 1!\n"); + process.exit(1); + } + + if(packageLock.dependencies !== undefined) { + augmentDependencies(".", packageLock.dependencies); + } + } + ''; + }; + + # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes + reconstructPackageLock = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var lockObj = { + name: packageObj.name, + version: packageObj.version, + lockfileVersion: 1, + requires: true, + dependencies: {} + }; + + function augmentPackageJSON(filePath, dependencies) { + var packageJSON = path.join(filePath, "package.json"); + if(fs.existsSync(packageJSON)) { + var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + dependencies[packageObj.name] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: {} + }; + processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + } + } + + function processDependencies(dir, dependencies) { + if(fs.existsSync(dir)) { + var files = fs.readdirSync(dir); + + files.forEach(function(entry) { + var filePath = path.join(dir, entry); + var stats = fs.statSync(filePath); + + if(stats.isDirectory()) { + if(entry.substr(0, 1) == "@") { + // When we encounter a namespace folder, augment all packages belonging to the scope + var pkgFiles = fs.readdirSync(filePath); + + pkgFiles.forEach(function(entry) { + if(stats.isDirectory()) { + var pkgFilePath = path.join(filePath, entry); + augmentPackageJSON(pkgFilePath, dependencies); + } + }); + } else { + augmentPackageJSON(filePath, dependencies); + } + } + }); + } + } + + processDependencies("node_modules", lockObj.dependencies); + + fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); + ''; + }; + + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: + let + forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; + in + '' + # Pinpoint the versions of all dependencies to the ones that are actually being used + echo "pinpointing versions of dependencies..." + source $pinpointDependenciesScriptPath + + # Patch the shebangs of the bundled modules to prevent them from + # calling executables outside the Nix store as much as possible + patchShebangs . + + # Deploy the Node.js package by running npm install. Since the + # dependencies have been provided already by ourselves, it should not + # attempt to install them again, which is good, because we want to make + # it Nix's responsibility. If it needs to install any dependencies + # anyway (e.g. because the dependency parameters are + # incomplete/incorrect), it fails. + # + # The other responsibilities of NPM are kept -- version checks, build + # steps, postprocessing etc. + + export HOME=$TMPDIR + cd "${packageName}" + runHook preRebuild + + ${stdenv.lib.optionalString bypassCache '' + ${stdenv.lib.optionalString reconstructLock '' + if [ -f package-lock.json ] + then + echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" + echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" + rm package-lock.json + else + echo "No package-lock.json file found, reconstructing..." + fi + + node ${reconstructPackageLock} + ''} + + node ${addIntegrityFieldsScript} + ''} + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} rebuild + + if [ "''${dontNpmInstall-}" != "1" ] + then + # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. + rm -f npm-shrinkwrap.json + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${stdenv.lib.optionalString production "--production"} install + fi + ''; + + # Builds and composes an NPM package including all its dependencies + buildNodePackage = + { name + , packageName + , version + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , preRebuild ? "" + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" ]; + in + stdenv.mkDerivation ({ + name = "node_${name}-${version}"; + buildInputs = [ tarWrapper python nodejs ] + ++ stdenv.lib.optional (stdenv.isLinux) utillinux + ++ stdenv.lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit nodejs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall preRebuild unpackPhase buildPhase; + + compositionScript = composePackage args; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; + + installPhase = '' + # Create and enter a root node_modules/ folder + mkdir -p $out/lib/node_modules + cd $out/lib/node_modules + + # Compose the package and all its dependencies + source $compositionScriptPath + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Create symlink to the deployed executable folder, if applicable + if [ -d "$out/lib/node_modules/.bin" ] + then + ln -s $out/lib/node_modules/.bin $out/bin + fi + + # Create symlinks to the deployed manual page folders, if applicable + if [ -d "$out/lib/node_modules/${packageName}/man" ] + then + mkdir -p $out/share + for dir in "$out/lib/node_modules/${packageName}/man/"* + do + mkdir -p $out/share/man/$(basename "$dir") + for page in "$dir"/* + do + ln -s $page $out/share/man/$(basename "$dir") + done + done + fi + + # Run post install hook, if provided + runHook postInstall + ''; + } // extraArgs); + + # Builds a development shell + buildNodeShell = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; + + nodeDependencies = stdenv.mkDerivation ({ + name = "node-dependencies-${name}-${version}"; + + buildInputs = [ tarWrapper python nodejs ] + ++ stdenv.lib.optional (stdenv.isLinux) utillinux + ++ stdenv.lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall unpackPhase buildPhase; + + includeScript = includeDependencies { inherit dependencies; }; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; + + installPhase = '' + mkdir -p $out/${packageName} + cd $out/${packageName} + + source $includeScriptPath + + # Create fake package.json to make the npm commands work properly + cp ${src}/package.json . + chmod 644 package.json + ${stdenv.lib.optionalString bypassCache '' + if [ -f ${src}/package-lock.json ] + then + cp ${src}/package-lock.json . + fi + ''} + + # Go to the parent folder to make sure that all packages are pinpointed + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Expose the executables that were installed + cd .. + ${stdenv.lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + mv ${packageName} lib + ln -s $out/lib/node_modules/.bin $out/bin + ''; + } // extraArgs); + in + stdenv.mkDerivation { + name = "node-shell-${name}-${version}"; + + buildInputs = [ python nodejs ] ++ stdenv.lib.optional (stdenv.isLinux) utillinux ++ buildInputs; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/shell < |##| | | (o \`--' //`-----------------------------' diff --git a/testutils/poc/kapow b/testutils/poc/kapow index c3a4519d..7182d689 100755 --- a/testutils/poc/kapow +++ b/testutils/poc/kapow @@ -1,4 +1,7 @@ -#!/usr/bin/env python +#! /usr/bin/env nix-shell +#! nix-shell -i python3.7 -p python37 python37Packages.aiohttp python37Packages.requests python37Packages.click +# +# TODO: maybe add an option (cli) to supply the external address # # Copyright 2019 Banco Bilbao Vizcaya Argentaria, S.A. @@ -20,16 +23,28 @@ from collections import namedtuple from urllib.parse import urlparse from uuid import uuid4 import asyncio +import binascii +import contextlib +import datetime import io +import ipaddress import json import logging import os import shlex import ssl import sys +import tempfile +import uuid from aiohttp import web, StreamReader from aiohttp.web_urldispatcher import UrlDispatcher +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import serialization +from cryptography import x509 +from cryptography.x509.oid import NameOID import click import requests @@ -38,6 +53,79 @@ log = logging.getLogger('kapow') loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) +KAPOW_CONTROL_URL="https://localhost:8081" +KAPOW_DATA_URL="http://localhost:8082" + +######################################################################## +# HTTPS Management # +######################################################################## + +def generate_ssl_cert(name, alt=None): + # Generate our key + key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + # Various details about who we are. For a self-signed certificate the + # subject and issuer are always the same. + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, name), + ]) + + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=3650) + ) + + if alt is not None: + try: + ip = ipaddress.ip_address(alt) + except: + cert = cert.add_extension( + x509.SubjectAlternativeName([x509.DNSName(alt)]), + critical=True, + ) + else: + cert = cert.add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ip)]), + critical=True, + ) + finally: + cert = cert.add_extension( + x509.ExtendedKeyUsage( + [x509.oid.ExtendedKeyUsageOID.SERVER_AUTH], + ), + critical=True + ) + else: + cert=cert.add_extension( + x509.ExtendedKeyUsage( + [x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH], + ), + critical=True + ) + + cert = cert.sign(key, hashes.SHA256()) + + key_bytes = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + crt_bytes = cert.public_bytes(serialization.Encoding.PEM) + + return (key_bytes, crt_bytes) + + ######################################################################## # Resource Management # ######################################################################## @@ -255,8 +343,7 @@ def handle_route(entrypoint, command): shell_task = await asyncio.create_subprocess_shell( args, env={**os.environ, - "KAPOW_DATA_URL": "http://localhost:8081", - "KAPOW_CONTROL_URL": "http://localhost:8081", + "KAPOW_DATA_URL": KAPOW_DATA_URL, "KAPOW_HANDLER_ID": id }, stdin=asyncio.subprocess.DEVNULL) @@ -279,7 +366,7 @@ def handle_route(entrypoint, command): def error_body(reason): - return {"reason": reason, "foo": "bar"} + return {"reason": reason} def get_routes(app): async def _get_routes(request): @@ -399,41 +486,31 @@ def delete_route(app): # aiohttp webapp # ######################################################################## +async def report_result(proc): + await proc.communicate() + print(f"Process exited with code {proc.returncode}") + async def run_init_script(app, scripts, interactive): """ Run the init script if given, then wait for the shell to finish. """ - if not scripts: - # No script given - if not interactive: - return - else: - cmd = "/bin/bash" - else: - def build_filenames(): - for filename in scripts: - yield shlex.quote(filename) - yield "<(echo)" - filenames = " ".join(build_filenames()) - if interactive: - cmd = f"/bin/bash --init-file <(cat {filenames})" + for script in scripts: + try: + result = await asyncio.create_subprocess_exec( + script, + env={**os.environ, + "KAPOW_CONTROL_CLIENT_CERT": app["client_cert"], + "KAPOW_CONTROL_CLIENT_KEY": app["client_key"], + "KAPOW_CONTROL_SERVER_CERT": app["server_cert"], + "KAPOW_CONTROL_URL": KAPOW_CONTROL_URL, + }) + except Exception as exc: + print(exc) else: - cmd = f"/bin/bash <(cat {filenames})" - - shell_task = await asyncio.create_subprocess_shell( - cmd, - executable="/bin/bash", - env={**os.environ, - "KAPOW_DATA_URL": "http://localhost:8081", - "KAPOW_CONTROL_URL": "http://localhost:8081" - }) + asyncio.create_task(report_result(result)) - await shell_task.wait() - if interactive: - await app.cleanup() - os._exit(shell_task.returncode) class InvalidRouteError(Exception): @@ -481,7 +558,28 @@ async def start_background_tasks(app): app["debug_tasks"] = loop.create_task(run_init_script(app, app["scripts"], app["interactive"])) -async def start_kapow_server(bind, scripts, certfile=None, interactive=False, keyfile=None): +def reduce_addr(addr): + """Drop the port part from an `addr:port` string (IPv6 aware)""" + addr, *_ = addr.rsplit(':', 1) + if addr.startswith('[') and addr.endswith(']'): + return addr[1:-1] + else: + return addr + + +async def start_kapow_server(user_bind, + control_bind, + data_bind, + scripts, + certfile=None, + interactive=False, + keyfile=None, + control_reachable_addr="localhost:8081"): + global KAPOW_CONTROL_URL + KAPOW_CONTROL_URL=f"https://{control_reachable_addr}" + # + # USER + # user_app = DynamicApplication(client_max_size=1024**3) user_app["user_routes"] = list() # [KapowRoute] user_runner = web.AppRunner(user_app) @@ -492,32 +590,73 @@ async def start_kapow_server(bind, scripts, certfile=None, interactive=False, ke ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) ssl_context.load_cert_chain(certfile, keyfile) - ip, port = bind.split(':') - user_site = web.TCPSite(user_runner, ip, int(port), ssl_context=ssl_context) + user_ip, user_port = user_bind.rsplit(':', 2) + user_site = web.TCPSite(user_runner, user_ip, int(user_port), + ssl_context=ssl_context) await user_site.start() + # + # CONTROL + # + alternate_name = reduce_addr(control_reachable_addr) + srv_key_bytes, srv_crt_bytes = generate_ssl_cert("control", alternate_name) + cli_key_bytes, cli_crt_bytes = generate_ssl_cert("control") + + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + with tempfile.NamedTemporaryFile(suffix=".pem", delete=True) as pem_file, \ + tempfile.NamedTemporaryFile(suffix=".key", delete=True) as key_file, \ + tempfile.NamedTemporaryFile(suffix=".pem", delete=True) as cli_crt_file: + pem_file.write(srv_crt_bytes) + pem_file.flush() + key_file.write(srv_key_bytes) + key_file.flush() + cli_crt_file.write(cli_crt_bytes) + cli_crt_file.flush() + + context.verify_mode = ssl.CERT_REQUIRED + context.load_cert_chain(pem_file.name, key_file.name) + context.load_verify_locations(cafile=cli_crt_file.name) + control_app = web.Application(client_max_size=1024**3) control_app.add_routes([ - # Control API web.get('/routes', get_routes(user_app)), web.get('/routes/{id}', get_route(user_app)), web.post('/routes', append_route(user_app)), web.put('/routes', insert_route(user_app)), web.delete('/routes/{id}', delete_route(user_app)), - - # Data API - web.get('/handlers/{id}/{field:.*}', get_field), - web.put('/handlers/{id}/{field:.*}', set_field), ]) control_app["scripts"] = scripts + control_app["client_cert"] = cli_crt_bytes + control_app["client_key"] = cli_key_bytes + control_app["server_cert"] = srv_crt_bytes control_app["interactive"] = interactive control_app.on_startup.append(start_background_tasks) control_runner = web.AppRunner(control_app) + await control_runner.setup() - control_site = web.TCPSite(control_runner, '127.0.0.1', 8081) + + control_ip, control_port = control_bind.rsplit(':', 2) + control_site = web.TCPSite(control_runner, control_ip, + int(control_port), ssl_context=context) await control_site.start() + # + # DATA + # + data_app = web.Application(client_max_size=1024**3) + data_app.add_routes([ + # Data API + web.get('/handlers/{id}/{field:.*}', get_field), + web.put('/handlers/{id}/{field:.*}', set_field), + ]) + + data_runner = web.AppRunner(data_app) + + await data_runner.setup() + data_ip, data_port = data_bind.rsplit(':', 2) + data_site = web.TCPSite(data_runner, data_ip, int(data_port)) + await data_site.start() ######################################################################## @@ -536,13 +675,25 @@ def kapow(ctx): @click.option("--certfile", default=None) @click.option("--keyfile", default=None) @click.option("--bind", default="0.0.0.0:8080") +@click.option("--control-bind", default="0.0.0.0:8081") +@click.option("--data-bind", default="0.0.0.0:8082") +@click.option("--control-reachable-addr", default="localhost:8081") @click.option("-i", "--interactive", is_flag=True) @click.argument("scripts", nargs=-1) -def server(certfile, keyfile, bind, interactive, scripts): +def server(certfile, keyfile, bind, interactive, scripts, + control_reachable_addr, control_bind, data_bind): if bool(certfile) ^ bool(keyfile): print("For SSL both 'certfile' and 'keyfile' should be provided.") sys.exit(1) - loop.run_until_complete(start_kapow_server(bind, scripts, certfile, interactive, keyfile)) + loop.run_until_complete( + start_kapow_server(bind, + control_bind, + data_bind, + scripts, + certfile, + interactive, + keyfile, + control_reachable_addr)) loop.run_forever() @kapow.group(help="Manage current server HTTP routes") @@ -550,59 +701,79 @@ def route(): pass +@contextlib.contextmanager +def kapow_control_certs(): + with tempfile.NamedTemporaryFile(suffix='.crt', encoding='utf-8', mode='w') as srv_cert, \ + tempfile.NamedTemporaryFile(suffix='.crt', encoding='utf-8', mode='w') as cli_cert, \ + tempfile.NamedTemporaryFile(suffix='.key', encoding='utf-8', mode='w') as cli_key: + srv_cert.write(os.environ["KAPOW_CONTROL_SERVER_CERT"]) + srv_cert.file.flush() + cli_cert.write(os.environ["KAPOW_CONTROL_CLIENT_CERT"]) + cli_cert.file.flush() + cli_key.write(os.environ["KAPOW_CONTROL_CLIENT_KEY"]) + cli_key.file.flush() + session=requests.Session() + session.verify=srv_cert.name + session.cert=(cli_cert.name, cli_key.name) + yield session + + @route.command("add") @click.option("-c", "--command", nargs=1) @click.option("-e", "--entrypoint", default="/bin/sh -c") @click.option("-X", "--method", default="GET") -@click.option("--url", envvar='KAPOW_CONTROL_URL') +@click.option("--url", envvar='KAPOW_CONTROL_URL', default=KAPOW_CONTROL_URL) @click.argument("url_pattern", nargs=1) @click.argument("command_file", required=False) def route_add(url_pattern, entrypoint, command, method, url, command_file): - if command: - # Command is given inline - source = command - elif command_file is None: - # No command - source = "" - elif command_file == '-': - # Read commands from stdin - source = sys.stdin.read() - else: - # Read commands from a file - with open(command_file, 'r', encoding='utf-8') as handler: - source = handler.read() - - response = requests.post(f"{url}/routes", - json={"method": method, - "url_pattern": url_pattern, - "entrypoint": entrypoint, - "command": source}) - response.raise_for_status() - print(json.dumps(response.json(), indent=2)) + with kapow_control_certs() as requests: + if command: + # Command is given inline + source = command + elif command_file is None: + # No command + source = "" + elif command_file == '-': + # Read commands from stdin + source = sys.stdin.read() + else: + # Read commands from a file + with open(command_file, 'r', encoding='utf-8') as handler: + source = handler.read() + + response = requests.post(f"{url}/routes", + json={"method": method, + "url_pattern": url_pattern, + "entrypoint": entrypoint, + "command": source}) + response.raise_for_status() + print(json.dumps(response.json(), indent=2)) @route.command("remove") -@click.option("--url", envvar='KAPOW_CONTROL_URL') +@click.option("--url", envvar='KAPOW_CONTROL_URL', default=KAPOW_CONTROL_URL) @click.argument("route-id") def route_remove(route_id, url): - response = requests.delete(f"{url}/routes/{route_id}") - response.raise_for_status() + with kapow_control_certs() as requests: + response = requests.delete(f"{url}/routes/{route_id}") + response.raise_for_status() @route.command("list") -@click.option("--url", envvar='KAPOW_CONTROL_URL') +@click.option("--url", envvar='KAPOW_CONTROL_URL', default=KAPOW_CONTROL_URL) @click.argument("route-id", nargs=1, required=False, default=None) def route_list(route_id, url): - if route_id is None: - response = requests.get(f"{url}/routes") - else: - response = requests.get(f"{url}/routes/{route_id}") - response.raise_for_status() - print(json.dumps(response.json(), indent=2)) + with kapow_control_certs() as requests: + if route_id is None: + response = requests.get(f"{url}/routes") + else: + response = requests.get(f"{url}/routes/{route_id}") + response.raise_for_status() + print(json.dumps(response.json(), indent=2)) @kapow.command("set", help="Set data from the current context") -@click.option("--url", envvar='KAPOW_DATA_URL') +@click.option("--url", envvar='KAPOW_DATA_URL', default=KAPOW_DATA_URL) @click.option("--handler-id", envvar='KAPOW_HANDLER_ID') @click.argument("path", nargs=1) @click.argument("value", required=False) @@ -622,7 +793,7 @@ def kapow_set(url, handler_id, path, value): @kapow.command("get", help="Get data from the current context") -@click.option("--url", envvar='KAPOW_DATA_URL') +@click.option("--url", envvar='KAPOW_DATA_URL', default=KAPOW_DATA_URL) @click.option("--handler-id", envvar='KAPOW_HANDLER_ID') @click.argument("path", nargs=1) def kapow_get(url, handler_id, path):