Skip to content

Commit

Permalink
Standalone API server with integration tests
Browse files Browse the repository at this point in the history
* Tried to separate out the API server options that require kubernetes API configuration options.
* Don't apply those command line options if `--standalone-mode` has been supplied.
* Added a wrapper around the Kubernetes integration test framework to also launch the Navigator API server and connect it to the same Etcd server.
* Currently parses stderr to check for a successful startup message, but the API server ought to have an `/healthz` endpoint.

Fixes: jetstack#364

Hook up a navigator cli helper
  • Loading branch information
wallrj committed May 21, 2018
1 parent 80ade4c commit 67a15ce
Show file tree
Hide file tree
Showing 10 changed files with 593 additions and 18 deletions.
16 changes: 14 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ CMDS := controller apiserver pilot-elasticsearch pilot-cassandra
GOPATH ?= /tmp/go
GOFLAGS ?= "-a"

# Path of the Navigator API server for use in integration tests
TEST_ASSET_NAVIGATOR_APISERVER ?= "${CURDIR}/navigator-apiserver_linux_amd64"

help:
# all - runs verify, build and docker_build targets
# test - runs go_test target
Expand All @@ -44,7 +47,7 @@ build: $(CMDS)

generate: .generate_files

verify: .hack_verify dep_verify go_verify helm_verify
verify: .hack_verify dep_verify go_verify helm_verify test_integration

.hack_verify:
@echo Running repo-infra verify scripts
Expand Down Expand Up @@ -90,7 +93,7 @@ $(CMDS):
go_build: $(CMDS)

go_test:
go test -v $$(go list ./... | grep -v '/vendor/')
go test -v $$(go list ./... | grep -v -e '/vendor/' -e 'github.com/jetstack/navigator/test/')

go_fmt:
./hack/verify-lint.sh
Expand All @@ -105,3 +108,12 @@ go_fmt:
# Helm targets
helm_verify:
helm lint contrib/charts/*

.download_integration_test_binaries:
mkdir -p vendor/sigs.k8s.io/testing_frameworks/integration/assets/bin
DEBUG=1 vendor/sigs.k8s.io/testing_frameworks/integration/scripts/download-binaries.sh
touch .download_integration_test_binaries

test_integration: .download_integration_test_binaries apiserver
TEST_ASSET_NAVIGATOR_APISERVER=$(TEST_ASSET_NAVIGATOR_APISERVER) \
go test -v ./test/integration/...
57 changes: 41 additions & 16 deletions cmd/apiserver/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,20 +130,47 @@ func (o NavigatorServerOptions) Config() (*apiserver.Config, error) {
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
}

serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err
config := &apiserver.Config{
GenericConfig: genericapiserver.NewRecommendedConfig(apiserver.Codecs),
}

client, err := clientset.NewForConfig(serverConfig.LoopbackClientConfig)
if err != nil {
// Do not call RecommendedOptions.ApplyTo because some of its sub-options
// require kube-apiserver configuration options
if err := o.RecommendedOptions.Etcd.ApplyTo(&config.GenericConfig.Config); err != nil {
return nil, err
}
if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.GenericConfig.Config); err != nil {
return nil, err
}
if err := o.RecommendedOptions.Audit.ApplyTo(&config.GenericConfig.Config); err != nil {
return nil, err
}
if err := o.RecommendedOptions.Features.ApplyTo(&config.GenericConfig.Config); err != nil {
return nil, err
}
sharedInformers := informers.NewSharedInformerFactory(client, serverConfig.LoopbackClientConfig.Timeout)

// only enable admission control when running in-cluster as we require a
// kubernetes client
if !o.StandaloneMode {
// These RecommendedOptions require kube-apiserver configuration and
// won't work in a standalone API server
if err := o.RecommendedOptions.Authentication.ApplyTo(&config.GenericConfig.Config); err != nil {
return nil, err
}
if err := o.RecommendedOptions.Authorization.ApplyTo(&config.GenericConfig.Config); err != nil {
return nil, err
}
if err := o.RecommendedOptions.CoreAPI.ApplyTo(config.GenericConfig); err != nil {
return nil, err
}

client, err := clientset.NewForConfig(config.GenericConfig.LoopbackClientConfig)
if err != nil {
return nil, err
}
sharedInformers := informers.NewSharedInformerFactory(client, config.GenericConfig.LoopbackClientConfig.Timeout)
config.SharedInformerFactory = sharedInformers

inClusterConfig, err := restclient.InClusterConfig()
if err != nil {
glog.Errorf("Failed to get kube client config: %v", err)
Expand All @@ -158,18 +185,14 @@ func (o NavigatorServerOptions) Config() (*apiserver.Config, error) {
}

kubeSharedInformers := kubeinformers.NewSharedInformerFactory(kubeClient, 10*time.Minute)
serverConfig.SharedInformerFactory = kubeSharedInformers
config.GenericConfig.SharedInformerFactory = kubeSharedInformers

serverConfig.AdmissionControl, err = buildAdmission(&o, client, sharedInformers, kubeClient, kubeSharedInformers)
config.GenericConfig.AdmissionControl, err = buildAdmission(&o, client, sharedInformers, kubeClient, kubeSharedInformers)
if err != nil {
return nil, fmt.Errorf("failed to initialize admission: %v", err)
}
}

config := &apiserver.Config{
GenericConfig: serverConfig,
SharedInformerFactory: sharedInformers,
}
return config, nil
}

Expand Down Expand Up @@ -201,10 +224,12 @@ func (o NavigatorServerOptions) RunNavigatorServer(stopCh <-chan struct{}) error
return err
}

server.GenericAPIServer.AddPostStartHook("start-navigator-server-informers", func(context genericapiserver.PostStartHookContext) error {
config.SharedInformerFactory.Start(context.StopCh)
return nil
})
if !o.StandaloneMode {
server.GenericAPIServer.AddPostStartHook("start-navigator-server-informers", func(context genericapiserver.PostStartHookContext) error {
config.SharedInformerFactory.Start(context.StopCh)
return nil
})
}

return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
132 changes: 132 additions & 0 deletions internal/test/integration/framework/framework.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package framework

import (
"fmt"
"io"
"net/url"
"path/filepath"
"time"

"sigs.k8s.io/testing_frameworks/integration"

"github.com/jetstack/navigator/internal/test/integration/framework/internal"
)

type NavigatorAPIServer struct {
URL *url.URL
Path string
Args []string
StartTimeout time.Duration
StopTimeout time.Duration
CertDir string
EtcdURL *url.URL
APIServerURL *url.URL
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}

func (s *NavigatorAPIServer) Start() error {
if s.EtcdURL == nil {
return fmt.Errorf("expected EtcdURL to be configured")
}

var err error

s.processState = &internal.ProcessState{}

s.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"navigator-apiserver",
s.URL,
s.CertDir,
s.Path,
s.StartTimeout,
s.StopTimeout,
)
if err != nil {
return err
}

// s.processState.HealthCheckEndpoint = "/healthz"
s.processState.StartMessage = "Serving securely on 127.0.0.1"
s.URL = &s.processState.URL
s.CertDir = s.processState.Dir
s.Path = s.processState.Path
s.StartTimeout = s.processState.StartTimeout
s.StopTimeout = s.processState.StopTimeout

s.processState.Args, err = internal.RenderTemplates(
append(
internal.DoAPIServerArgDefaulting(nil),
s.Args...,
),
s,
)
if err != nil {
return err
}
return s.processState.Start(s.Out, s.Err)
}

func (s *NavigatorAPIServer) Stop() error {
return s.processState.Stop()
}

type NavigatorControlPlane struct {
*integration.ControlPlane
NavigatorAPIServer *NavigatorAPIServer
}

func (f *NavigatorControlPlane) Start() error {
if f.ControlPlane == nil {
f.ControlPlane = &integration.ControlPlane{}
}
err := f.ControlPlane.Start()
if err != nil {
return err
}
if f.NavigatorAPIServer == nil {
f.NavigatorAPIServer = &NavigatorAPIServer{}
}
f.NavigatorAPIServer.EtcdURL = f.Etcd.URL
f.NavigatorAPIServer.APIServerURL = f.APIServer.URL
return f.NavigatorAPIServer.Start()
}

func (f *NavigatorControlPlane) Stop() error {
if f.NavigatorAPIServer != nil {
err := f.NavigatorAPIServer.Stop()
if err != nil {
return err
}
}
return f.ControlPlane.Stop()
}

func (f *NavigatorControlPlane) NavigatorAPIURL() *url.URL {
return &url.URL{
Scheme: "https",
Host: f.NavigatorAPIServer.URL.Host,
}
}

func (f NavigatorControlPlane) NavigatorCtl() *NavigatorCtl {
return &NavigatorCtl{
&integration.KubeCtl{
Opts: []string{
"--server",
f.NavigatorAPIURL().String(),
"--client-certificate",
filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.crt"),
"--client-key",
filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.key"),
"--certificate-authority",
filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.crt"),
},
},
}
}

type NavigatorCtl struct {
*integration.KubeCtl
}
53 changes: 53 additions & 0 deletions internal/test/integration/framework/internal/address_manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package internal

import (
"fmt"
"net"
)

// AddressManager allocates a new address (interface & port) a process
// can bind and keeps track of that.
type AddressManager struct {
port int
host string
}

// Initialize returns a address a process can listen on. It returns
// a tuple consisting of a free port and the hostname resolved to its IP.
func (d *AddressManager) Initialize() (port int, resolvedHost string, err error) {
if d.port != 0 {
return 0, "", fmt.Errorf("this AddressManager is already initialized")
}
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return
}
d.port = l.Addr().(*net.TCPAddr).Port
defer func() {
err = l.Close()
}()
d.host = addr.IP.String()
return d.port, d.host, nil
}

// Port returns the port that this AddressManager is managing. Port returns an
// error if this AddressManager has not yet been initialized.
func (d *AddressManager) Port() (int, error) {
if d.port == 0 {
return 0, fmt.Errorf("this AdressManager is not initialized yet")
}
return d.port, nil
}

// Host returns the host that this AddressManager is managing. Host returns an
// error if this AddressManager has not yet been initialized.
func (d *AddressManager) Host() (string, error) {
if d.host == "" {
return "", fmt.Errorf("this AdressManager is not initialized yet")
}
return d.host, nil
}
17 changes: 17 additions & 0 deletions internal/test/integration/framework/internal/apiserver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package internal

var APIServerDefaultArgs = []string{
"-v=6",
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--secure-port={{ if .URL }}{{ .URL.Port }}{{ end }}",
"--bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}",
}

func DoAPIServerArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}

return APIServerDefaultArgs
}
28 changes: 28 additions & 0 deletions internal/test/integration/framework/internal/arguments.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package internal

import (
"bytes"
"html/template"
)

func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) {
var t *template.Template

for _, arg := range argTemplates {
t, err = template.New(arg).Parse(arg)
if err != nil {
args = nil
return
}

buf := &bytes.Buffer{}
err = t.Execute(buf, data)
if err != nil {
args = nil
return
}
args = append(args, buf.String())
}

return
}
Loading

0 comments on commit 67a15ce

Please sign in to comment.