-
Notifications
You must be signed in to change notification settings - Fork 31
Pilot executes cassandra
directly, bypassing the container image entry point
#347
base: master
Are you sure you want to change the base?
Changes from 1 commit
69a59e5
20deca2
350c038
330437b
2426e2c
a25029d
74bb244
641310c
015c813
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
#!/bin/bash | ||
# | ||
# Download the binaries needed by the sigs.k8s.io/testing_frameworks/integration package. | ||
# etcd, kube-apiserver, kubectl | ||
# XXX: There is already a script to do this: | ||
# * sigs.k8s.io/testing_frameworks/integration/scripts/download-binaries.sh | ||
# But it currently downloads kube-apiserver v1.10.0-alpha.1 which doesn't support ``CustomResourceSubresources``. | ||
# See https://github.com/kubernetes-sigs/testing_frameworks/issues/44 | ||
|
||
set -o errexit | ||
set -o nounset | ||
set -o pipefail | ||
set -o xtrace | ||
|
||
# Close stdin | ||
exec 0<&- | ||
|
||
ROOT_DIR="$(git rev-parse --show-toplevel)" | ||
|
||
ETCD_VERSION=v3.2.10 | ||
ETCD_URL="https://storage.googleapis.com/etcd" | ||
|
||
KUBE_VERSION_URL="https://storage.googleapis.com/kubernetes-release/release/stable-1.10.txt" | ||
KUBE_VERSION=$(curl --fail --silent "${KUBE_VERSION_URL}") | ||
KUBE_BIN_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64" | ||
|
||
ASSETS_DIR="${ROOT_DIR}/vendor/sigs.k8s.io/testing_frameworks/integration/assets/bin" | ||
|
||
mkdir -p "${ASSETS_DIR}" | ||
|
||
curl --fail --silent ${ETCD_URL}/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | \ | ||
tar --extract --gzip --directory="${ASSETS_DIR}" --strip-components=1 --wildcards '*/etcd' | ||
curl --fail --silent --output "${ASSETS_DIR}/kube-apiserver" "${KUBE_BIN_URL}/kube-apiserver" | ||
curl --fail --silent --output "${ASSETS_DIR}/kubectl" "${KUBE_BIN_URL}/kubectl" | ||
|
||
chmod +x ${ASSETS_DIR}/* |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,200 @@ | ||
package integration_test | ||
|
||
import ( | ||
"os" | ||
"os/exec" | ||
"path/filepath" | ||
"syscall" | ||
"testing" | ||
"time" | ||
|
||
"github.com/onsi/gomega/gbytes" | ||
"github.com/pkg/errors" | ||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
"sigs.k8s.io/testing_frameworks/integration" | ||
|
||
"github.com/jetstack/navigator/internal/test/util/testfs" | ||
) | ||
|
||
// MakeKubeConfig creates a kube/config file which is suitable for communicating | ||
// with the API server started by ``integration.ControlPlane``. | ||
func MakeKubeConfig(f *integration.ControlPlane, path string) error { | ||
ctl := f.KubeCtl() | ||
ctl.Opts = []string{"--kubeconfig", path} | ||
|
||
_, _, err := ctl.Run( | ||
"config", | ||
"set-credentials", | ||
"user1", | ||
) | ||
if err != nil { | ||
return errors.Wrap(err, "unable to create user") | ||
} | ||
|
||
_, _, err = ctl.Run( | ||
"config", | ||
"set-cluster", | ||
"integration1", | ||
"--server", | ||
f.APIURL().String(), | ||
"--certificate-authority", | ||
filepath.Join(f.APIServer.CertDir, "apiserver.crt"), | ||
) | ||
if err != nil { | ||
return errors.Wrap(err, "unable to create cluster") | ||
} | ||
|
||
_, _, err = ctl.Run( | ||
"config", | ||
"set-context", | ||
"default", | ||
"--cluster", "integration1", | ||
"--user", "user1", | ||
) | ||
if err != nil { | ||
return errors.Wrap(err, "unable to create context") | ||
} | ||
|
||
_, _, err = ctl.Run( | ||
"config", | ||
"use-context", | ||
"default", | ||
) | ||
if err != nil { | ||
return errors.Wrap(err, "unable to use context") | ||
} | ||
return nil | ||
} | ||
|
||
func TestPilotCassandra(t *testing.T) { | ||
// Start the API server with CustomResourceSubresources feature enabled. | ||
// Navigator pilot calls on UpdateStatus on Pilot resources for example. | ||
cp := &integration.ControlPlane{ | ||
APIServer: &integration.APIServer{ | ||
Args: []string{ | ||
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}", | ||
"--cert-dir={{ .CertDir }}", | ||
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}", | ||
"--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}", | ||
"--secure-port=0", | ||
"--feature-gates=CustomResourceSubresources=true", | ||
"-v=4", "--alsologtostderr", | ||
}, | ||
// Out: os.Stdout, | ||
// Err: os.Stderr, | ||
}, | ||
} | ||
err := cp.Start() | ||
require.NoError(t, err) | ||
defer func() { | ||
err := cp.Stop() | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
}() | ||
cli := cp.KubeCtl() | ||
|
||
// Create a CRD for the Navigator Pilot resource type. | ||
// This allows us to test the Pilot against the Kubernetes API server, | ||
// without having to start the Navigator API server and configure aggregation etc. | ||
crdPath, err := filepath.Abs("testdata/pilot_crd.yaml") | ||
require.NoError(t, err) | ||
stdout, stderr, err := cli.Run( | ||
"apply", | ||
"--filename", | ||
crdPath, | ||
) | ||
t.Log("stdout", stdout) | ||
t.Log("stderr", stderr) | ||
require.NoError(t, err) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. From here up to the start of the function should be in some shared function - this is code that is not specific to this test, or Cassandra. Also, we should add some comments to this explaining why we are registering a CRD, and what is and isn't okay to test whilst using a CRD (because a lot of code for e.g. validation won't be run, we should clearly define bounds for what is and isn't okay to test). As we spoke about before, I think we should look to remove this code ASAP in favour of launching navigator-apiserver itself. Also, given what we're trying to test here, would it be possible to implement this whole test as a unit test? I'm not sure why we need to bring up the entire process to verify that a config file is written ❓ |
||
|
||
// Create a Pilot resource for our Pilot process to watch and update. | ||
stdout, stderr, err = cli.Run( | ||
"create", | ||
"namespace", | ||
"ns1", | ||
) | ||
t.Log("stdout", stdout) | ||
t.Log("stderr", stderr) | ||
require.NoError(t, err) | ||
pilotResourcePath, err := filepath.Abs("testdata/pilot.yaml") | ||
require.NoError(t, err) | ||
stdout, stderr, err = cli.Run( | ||
"apply", | ||
"--filename", | ||
pilotResourcePath, | ||
) | ||
t.Log("stdout", stdout) | ||
t.Log("stderr", stderr) | ||
require.NoError(t, err) | ||
|
||
// Temporary configuration directories. | ||
tfs := testfs.New(t) | ||
kubeConfig := tfs.TempPath("kube.config") | ||
cassConfig := tfs.TempDir("etc_cassandra") | ||
pilotConfigDir := tfs.TempDir("etc/pilot") | ||
// A fake cassandra binary for the Pilot to execute. | ||
cassPath, err := filepath.Abs("testdata/fake_cassandra") | ||
require.NoError(t, err) | ||
err = MakeKubeConfig(cp, kubeConfig) | ||
require.NoError(t, err) | ||
|
||
pilotPath, pilotPathFound := os.LookupEnv("TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA") | ||
if !pilotPathFound { | ||
t.Fatal( | ||
"Please set environment variable TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA " + | ||
"with the path to the navigator-pilot-cassandra binary under test.") | ||
} | ||
|
||
expectedClusterName := "cluster1" | ||
cmd := exec.Command( | ||
pilotPath, | ||
"--pilot-name", "pilot1", | ||
"--pilot-namespace", "ns1", | ||
"--kubeconfig", kubeConfig, | ||
"--v=4", "--alsologtostderr", | ||
"--leader-elect=false", | ||
"--cassandra-cluster-name", expectedClusterName, | ||
"--cassandra-config-path", cassConfig, | ||
"--cassandra-path", cassPath, | ||
"--cassandra-rack", "rack-for-test", | ||
"--cassandra-dc", "dc-for-test", | ||
"--config-dir", pilotConfigDir, | ||
) | ||
startDetectStream := gbytes.NewBuffer() | ||
// The fake Cassandra script echos "FAKE CASSANDRA" in a loop | ||
ready := startDetectStream.Detect("FAKE CASSANDRA") | ||
cmd.Stdout = startDetectStream | ||
cmd.Stderr = os.Stderr | ||
// This will return when the executable writes "FAKE CASSANDRA" to stdout. | ||
err = cmd.Start() | ||
require.NoError(t, err) | ||
defer func() { | ||
// XXX: Pilot doesn't respond to SIGKILL | ||
err := cmd.Process.Signal(os.Interrupt) | ||
require.NoError(t, err) | ||
err = cmd.Wait() | ||
if exiterr, ok := err.(*exec.ExitError); ok { | ||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { | ||
if status.ExitStatus() == 255 { | ||
// XXX Pilot should return 0 if it exits cleanly after a signal. | ||
return | ||
} | ||
} | ||
} | ||
require.NoError(t, err) | ||
}() | ||
select { | ||
case <-ready: | ||
case <-time.After(time.Second * 5): | ||
t.Fatal("timeout waiting for process to start") | ||
} | ||
// The pilot has executed the Cassandra sub-process, so it should already | ||
// have written the configuration files as part of the pre-start hook | ||
// mechanism. | ||
// We did not provide an existing cassandra.yaml or cassandra-rackdc.properties. | ||
// So we expect thos files to now exist. | ||
assert.FileExists(t, cassConfig+"/cassandra.yaml") | ||
assert.FileExists(t, cassConfig+"/cassandra-rackdc.properties") | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
foo: | ||
bar: org.apache.cassandra.locator.SimpleSeedProvider |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/bin/bash | ||
set -o errexit | ||
set -o pipefail | ||
|
||
while true; do | ||
echo "FAKE CASSANDRA" | ||
sleep 1 | ||
done |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
apiVersion: navigator.jetstack.io/v1alpha1 | ||
kind: Pilot | ||
metadata: | ||
name: pilot1 | ||
namespace: ns1 | ||
ownerReferences: | ||
- apiVersion: navigator.jetstack.io/v1alpha1 | ||
kind: CassandraCluster | ||
name: cluster1 | ||
uid: 23c88696-cf7b-11e7-9ec9-0a580a200927 | ||
controller: true |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
apiVersion: apiextensions.k8s.io/v1beta1 | ||
kind: CustomResourceDefinition | ||
metadata: | ||
name: pilots.navigator.jetstack.io | ||
spec: | ||
group: navigator.jetstack.io | ||
version: v1alpha1 | ||
scope: Namespaced | ||
names: | ||
plural: pilots | ||
singular: pilot | ||
kind: Pilot | ||
# categories is a list of grouped resources the custom resource belongs to. | ||
categories: | ||
- all | ||
subresources: | ||
status: {} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we pin this to a specific patch release of 1.10? Or otherwise, clearly print out the exact version returned here in the test logs, so we can debug failures caused by patch version changes.