Skip to content

Commit

Permalink
[WIP] Migrate to a non-global VM
Browse files Browse the repository at this point in the history 
Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>
  • Loading branch information
@jimmykarily
jimmykarily committed Apr 3, 2023
1 parent 042f683 commit bd7f175
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 79 deletions.
76 changes: 39 additions & 37 deletions tests/decentralized_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,11 @@ import (
)

var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), func() {
var vm VM

BeforeEach(func() {
EventuallyConnects()
_, vm = startVM()
vm.EventuallyConnects(1200)
})

AfterEach(func() {
Expand All @@ -27,7 +30,7 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
Context("live cd", func() {
It("has default service active", func() {
if isFlavor("alpine") {
out, _ := Sudo("rc-status")
out, _ := vm.Sudo("rc-status")
Expect(out).Should(ContainSubstring("kairos"))
Expect(out).Should(ContainSubstring("kairos-agent"))
} else {
Expand All @@ -36,30 +39,29 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
// return out
// }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token"))

out, _ := Sudo("systemctl status kairos")
out, _ := vm.Sudo("systemctl status kairos")
Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/kairos.service; enabled; vendor preset: disabled)"))
}
})
})

Context("install", func() {
It("to disk with custom config", func() {
err := Machine.SendFile(os.Getenv("CLOUD_INIT"), "/tmp/config.yaml", "0770")
err := vm.Scp(os.Getenv("CLOUD_INIT"), "/tmp/config.yaml", "0770")
Expect(err).ToNot(HaveOccurred())

out, _ := Sudo("kairos-agent manual-install --device auto /tmp/config.yaml")
out, _ := vm.Sudo("kairos-agent manual-install --device auto /tmp/config.yaml")
Expect(out).Should(ContainSubstring("Running after-install hook"))
fmt.Println(out)
Sudo("sync")
detachAndReboot()
vm.Sudo("sync")
vm.Reboot()
})
})

Context("first-boot", func() {

It("has default services on", func() {
if isFlavor("alpine") {
out, _ := Sudo("rc-status")
out, _ := vm.Sudo("rc-status")
Expect(out).Should(ContainSubstring("kairos"))
Expect(out).Should(ContainSubstring("kairos-agent"))
} else {
Expand All @@ -68,10 +70,10 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
// return out
// }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token"))

out, _ := Sudo("systemctl status kairos-agent")
out, _ := vm.Sudo("systemctl status kairos-agent")
Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/kairos-agent.service; enabled; vendor preset: disabled)"))

out, _ = Sudo("systemctl status systemd-timesyncd")
out, _ = vm.Sudo("systemctl status systemd-timesyncd")
Expect(out).Should(ContainSubstring("loaded (/usr/lib/systemd/system/systemd-timesyncd.service; enabled; vendor preset: disabled)"))
}
})
Expand All @@ -82,30 +84,30 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
}

By("checking entries", func() {
state, _ := Sudo("blkid -L COS_STATE")
state, _ := vm.Sudo("blkid -L COS_STATE")
state = strings.TrimSpace(state)
out, _ := Sudo("blkid")
out, _ := vm.Sudo("blkid")
fmt.Println(out)
out, _ = Sudo("mkdir -p /tmp/mnt/STATE")
out, _ = vm.Sudo("mkdir -p /tmp/mnt/STATE")
fmt.Println(out)
out, _ = Sudo("mount " + state + " /tmp/mnt/STATE")
out, _ = vm.Sudo("mount " + state + " /tmp/mnt/STATE")
fmt.Println(out)
out, _ = Sudo("cat /tmp/mnt/STATE/grubmenu")
out, _ = vm.Sudo("cat /tmp/mnt/STATE/grubmenu")
Expect(out).Should(ContainSubstring("Kairos remote recovery"))

grub, _ := Sudo("cat /tmp/mnt/STATE/grub_oem_env")
grub, _ := vm.Sudo("cat /tmp/mnt/STATE/grub_oem_env")
Expect(grub).Should(ContainSubstring("default_menu_entry=Kairos"))

Sudo("umount /tmp/mnt/STATE")
vm.Sudo("umount /tmp/mnt/STATE")
})
})

It("configure k3s", func() {
_, err := Machine.Command("cat /run/cos/live_mode")
_, err := vm.Sudo("cat /run/cos/live_mode")
Expect(err).To(HaveOccurred())
if isFlavor("alpine") {
Eventually(func() string {
out, _ := Sudo("sudo cat /var/log/kairos/agent.log")
out, _ := vm.Sudo("sudo cat /var/log/kairos/agent.log")
fmt.Println(out)
return out
}, 20*time.Minute, 1*time.Second).Should(
Expand All @@ -115,7 +117,7 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
))
} else {
Eventually(func() string {
out, _ := Sudo("systemctl status kairos-agent")
out, _ := vm.Sudo("systemctl status kairos-agent")
return out
}, 30*time.Minute, 1*time.Second).Should(
Or(
Expand All @@ -127,7 +129,7 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu

PIt("configure edgevpn", func() {
Eventually(func() string {
out, _ := Sudo("cat /etc/systemd/system.conf.d/edgevpn-kairos.env")
out, _ := vm.Sudo("cat /etc/systemd/system.conf.d/edgevpn-kairos.env")
return out
}, 1*time.Minute, 1*time.Second).Should(
And(
Expand All @@ -137,29 +139,29 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu

It("has default image sizes", func() {
for _, p := range []string{"active.img", "passive.img"} {
out, _ := Sudo(`stat -c "%s" /run/initramfs/cos-state/cOS/` + p)
out, _ := vm.Sudo(`stat -c "%s" /run/initramfs/cos-state/cOS/` + p)
Expect(out).Should(ContainSubstring("3145728000"))
}
})

It("propagate kubeconfig", func() {
Eventually(func() string {
out, _ := Machine.Command("kairos get-kubeconfig")
out, _ := vm.Sudo("kairos get-kubeconfig")
return out
}, 900*time.Second, 10*time.Second).Should(ContainSubstring("https:"))

Eventually(func() string {
Machine.Command("kairos get-kubeconfig > kubeconfig")
out, _ := Machine.Command("KUBECONFIG=kubeconfig kubectl get nodes -o wide")
vm.Sudo("kairos get-kubeconfig > kubeconfig")
out, _ := vm.Sudo("KUBECONFIG=kubeconfig kubectl get nodes -o wide")
return out
}, 900*time.Second, 10*time.Second).Should(ContainSubstring("Ready"))
})

It("has roles", func() {
uuid, _ := Machine.Command("kairos-agent uuid")
uuid, _ := vm.Sudo("kairos-agent uuid")
Expect(uuid).ToNot(Equal(""))
Eventually(func() string {
out, _ := Machine.Command("kairos role list")
out, _ := vm.Sudo("kairos role list")
return out
}, 900*time.Second, 10*time.Second).Should(And(
ContainSubstring(uuid),
Expand All @@ -172,7 +174,7 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu

It("has machines with different IPs", func() {
Eventually(func() string {
out, _ := Machine.Command(`curl http://localhost:8080/api/machines`)
out, _ := vm.Sudo(`curl http://localhost:8080/api/machines`)
return out
}, 900*time.Second, 10*time.Second).Should(And(
ContainSubstring("10.1.0.1"),
Expand All @@ -185,16 +187,16 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
Skip("DNS not working on alpine yet")
}
// FIXUP: DNS needs reboot to take effect
Reboot()
vm.Reboot()
Eventually(func() string {
Machine.Command(`curl -X POST http://localhost:8080/api/dns --header "Content-Type: application/json" -d '{ "Regex": "foo.bar", "Records": { "A": "2.2.2.2" } }'`)
out, _ := Machine.Command("ping -c 1 foo.bar")
vm.Sudo(`curl -X POST http://localhost:8080/api/dns --header "Content-Type: application/json" -d '{ "Regex": "foo.bar", "Records": { "A": "2.2.2.2" } }'`)
out, _ := vm.Sudo("ping -c 1 foo.bar")
return out
}, 900*time.Second, 10*time.Second).Should(And(
ContainSubstring("2.2.2.2"),
))
Eventually(func() string {
out, _ := Machine.Command("ping -c 1 google.com")
out, _ := vm.Sudo("ping -c 1 google.com")
return out
}, 900*time.Second, 10*time.Second).Should(And(
ContainSubstring("64 bytes from"),
Expand All @@ -204,15 +206,15 @@ var _ = Describe("kairos decentralized k8s test", Label("decentralized-k8s"), fu
It("upgrades to a specific version", func() {
version, _ := Machine.Command("source /etc/os-release; echo $VERSION")

out, _ := Sudo("kairos-agent upgrade --image quay.io/kairos/kairos-opensuse:v1.0.0-rc2-k3sv1.21.14-k3s1")
out, _ := vm.Sudo("kairos-agent upgrade --image quay.io/kairos/kairos-opensuse:v1.0.0-rc2-k3sv1.21.14-k3s1")
Expect(out).To(ContainSubstring("Upgrade completed"))

Sudo("sync")
Reboot()
vm.Sudo("sync")
vm.Reboot()

EventuallyConnects(700)

version2, _ := Machine.Command("source /etc/os-release; echo $VERSION")
version2, _ := vm.Sudo("source /etc/os-release; echo $VERSION")
Expect(version).ToNot(Equal(version2))
})
})
Expand Down
117 changes: 75 additions & 42 deletions tests/tests_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
"testing"
Expand Down Expand Up @@ -83,61 +84,93 @@ func pass() string {
return pass
}

var _ = BeforeSuite(func() {

machineID := uuid.New().String()

func startVM() (context.Context, VM) {
if os.Getenv("ISO") == "" && os.Getenv("CREATE_VM") == "true" {
fmt.Println("ISO missing")
os.Exit(1)
}

if os.Getenv("CREATE_VM") == "true" {
t, err := ioutil.TempDir("", "")
Expect(err).ToNot(HaveOccurred())
var sshPort, spicePort int

p, _ := getFreePort()
sshPort = strconv.Itoa(p)
if os.Getenv("SSH_PORT") != "" {
sshPort = os.Getenv("SSH_PORT")
}
vmName := uuid.New().String()

opts := []types.MachineOption{
types.WithMemory("9000"),
types.WithISO(os.Getenv("ISO")),
types.WithSSHPort(sshPort),
types.WithID(machineID),
types.WithSSHUser(user()),
types.WithSSHPass(pass()),
types.OnFailure(func(p *process.Process) {
out, _ := ioutil.ReadFile(p.StdoutPath())
err, _ := ioutil.ReadFile(p.StderrPath())
status, _ := p.ExitCode()
fmt.Printf("VM Aborted: %s %s Exit status: %s", out, err, status)
Fail(fmt.Sprintf("VM Aborted: %s %s Exit status: %s", out, err, status))
}),
types.WithStateDir(t),
types.WithDataSource(os.Getenv("DATASOURCE")),
}
stateDir, err := os.MkdirTemp("", "")
Expect(err).ToNot(HaveOccurred())

if os.Getenv("USE_QEMU") == "true" {
opts = append(opts, types.QEMUEngine)
} else {
opts = append(opts, types.VBoxEngine)
}
sshPort, err = getFreePort()
Expect(err).ToNot(HaveOccurred())

m, err := machine.New(opts...)
if err != nil {
Fail(err.Error())
}
memory := os.Getenv("MEMORY")
if memory == "" {
memory = "2096"
}
cpus := os.Getenv("CPUS")
if cpus == "" {
cpus = "2"
}

Machine = m
opts := []types.MachineOption{
types.QEMUEngine,
types.WithISO(os.Getenv("ISO")),
types.WithMemory(memory),
types.WithCPU(cpus),
types.WithSSHPort(strconv.Itoa(sshPort)),
types.WithID(vmName),
types.WithSSHUser(user()),
types.WithSSHPass(pass()),
types.OnFailure(func(p *process.Process) {
out, _ := os.ReadFile(p.StdoutPath())
err, _ := os.ReadFile(p.StderrPath())
status, _ := p.ExitCode()

// We are explicitly killing the qemu process. We don't treat that as an error
// but we just print the output just in case.
fmt.Printf("\nVM Aborted: %s %s Exit status: %s\n", out, err, status)
}),
types.WithStateDir(stateDir),
types.WithDataSource(os.Getenv("DATASOURCE")),
}
if os.Getenv("KVM") != "" {
opts = append(opts, func(m *types.MachineConfig) error {
m.Args = append(m.Args,
"-enable-kvm",
)
return nil
})
}

if _, err := Machine.Create(context.Background()); err != nil {
Fail(err.Error())
if os.Getenv("USE_QEMU") == "true" {
opts = append(opts, types.QEMUEngine)

// You can connect to it with "spicy" or other tool.
// DISPLAY is already taken on Linux X sessions
if os.Getenv("MACHINE_SPICY") != "" {
spicePort, _ = getFreePort()
for spicePort == sshPort { // avoid collision
spicePort, _ = getFreePort()
}
display := fmt.Sprintf("-spice port=%d,addr=127.0.0.1,disable-ticketing=yes", spicePort)
opts = append(opts, types.WithDisplay(display))

cmd := exec.Command("spicy",
"-h", "127.0.0.1",
"-p", strconv.Itoa(spicePort))
err = cmd.Start()
Expect(err).ToNot(HaveOccurred())
}
} else {
opts = append(opts, types.VBoxEngine)
}
})
m, err := machine.New(opts...)
Expect(err).ToNot(HaveOccurred())

vm := NewVM(m, stateDir)

ctx, err := vm.Start(context.Background())
Expect(err).ToNot(HaveOccurred())

return ctx, vm
}

func getFreePort() (port int, err error) {
var a *net.TCPAddr
Expand Down

0 comments on commit bd7f175

Please sign in to comment.