From 37e0b57ac9584fed303a688ff95ee4f047d3902d Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Wed, 5 Jun 2024 10:35:14 +0200 Subject: [PATCH] Update e2e tests Make the e2e tests to expect an installed runtime version + endpoint and move them to the CRI-O CI to test the latest version. Signed-off-by: Sascha Grunert --- .github/workflows/crio.yml | 29 +++---- .github/workflows/e2e.yml | 79 ----------------- .gitignore | 1 + Makefile | 14 +-- test/e2e/events_test.go | 15 +--- test/e2e/help_test.go | 26 ++---- test/e2e/info_test.go | 16 +--- test/e2e/pull_test.go | 26 ++---- test/e2e/suite_test.go | 9 +- test/e2e/version_test.go | 9 +- test/framework/framework.go | 169 ++++++++---------------------------- 11 files changed, 78 insertions(+), 315 deletions(-) delete mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/crio.yml b/.github/workflows/crio.yml index 3e2659b089..6cd3a69a10 100644 --- a/.github/workflows/crio.yml +++ b/.github/workflows/crio.yml @@ -1,4 +1,4 @@ -name: critest CRI-O +name: CRI-O on: push: tags: @@ -7,14 +7,13 @@ on: - master pull_request: jobs: - # - # Run CRI tests against CRI-O - # - linux-build-and-critest-cri-o: + test: strategy: matrix: - version: [main] - name: ${{matrix.version}} / linux amd64 + suite: + - e2e + - critest + name: ${{matrix.suite}} runs-on: ubuntu-22.04 steps: - name: Install go @@ -50,21 +49,13 @@ jobs: sudo sysctl -w net.ipv4.ip_forward=1 sudo iptables -t nat -I POSTROUTING -s 127.0.0.0/8 ! -d 127.0.0.0/8 -j MASQUERADE - # enable criu support - sudo apt-get update - sudo apt-get install -y criu - - name: Install ginkgo run: | go install github.com/onsi/ginkgo/v2/ginkgo@latest ginkgo version sudo cp $(command -v ginkgo) /usr/local/bin - - name: Setup GCloud - uses: google-github-actions/setup-gcloud@v2 - - name: Install CRI-O latest main - if: ${{matrix.version == 'main'}} run: | curl https://raw.githubusercontent.com/cri-o/packaging/main/get | sudo bash @@ -88,6 +79,7 @@ jobs: working-directory: ${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools - name: Run critest + if: ${{matrix.suite == 'critest'}} run: | sudo -E PATH=$PATH critest \ --runtime-endpoint=unix:///var/run/crio/crio.sock \ @@ -95,6 +87,13 @@ jobs: --parallel=$(nproc) sudo journalctl -u crio > cri-o.log + - name: Run crictl e2e tests + if: ${{matrix.suite == 'e2e'}} + run: | + sudo -E PATH=$PATH make test-e2e \ + TESTFLAGS="-crictl-runtime-endpoint=unix://var/run/crio/crio.sock" + working-directory: ${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools + - name: Upload CRI-O logs uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index c5d5b092d9..0000000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: cri-tools e2e test -on: - push: - tags: - - "*" - branches: - - master - pull_request: -jobs: - test-e2e: - name: ${{ matrix.os }} - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - - strategy: - matrix: - os: [ubuntu-latest] - - steps: - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: '1.22' - cache: false - - - name: Set env - shell: bash - run: | - echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV - echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - - name: Cache go modules and build cache - uses: actions/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - '%LocalAppData%\go-build' # Windows - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-go- - - - name: Checkout - uses: actions/checkout@v4 - with: - path: src/github.com/kubernetes-sigs/cri-tools - - - name: Cache go modules and build cache - uses: actions/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - '%LocalAppData%\go-build' # Windows - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-go- - - - name: Run e2e test - run: | - sudo apt-get update &&\ - sudo apt-get install -y \ - conntrack \ - libseccomp-dev \ - libgpgme-dev - - VERSION=v1.0.1 &&\ - sudo mkdir -p /opt/cni/bin &&\ - sudo wget -qO- https://github.com/containernetworking/plugins/releases/download/$VERSION/cni-plugins-linux-amd64-$VERSION.tgz \ - | sudo tar xfz - -C /opt/cni/bin &&\ - ls -lah /opt/cni/bin - - VERSION=v1.1.0 &&\ - sudo wget -q -O \ - /usr/bin/runc \ - https://github.com/opencontainers/runc/releases/download/$VERSION/runc.amd64 &&\ - sudo chmod +x /usr/bin/runc &&\ - runc --version - - sudo -E PATH=$PATH make all install test-e2e TESTFLAGS=-v - working-directory: src/github.com/kubernetes-sigs/cri-tools diff --git a/.gitignore b/.gitignore index ae656d31ff..15e9a8aa20 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ Session.vim /build release-notes.md +test/e2e/e2e.test diff --git a/Makefile b/Makefile index f877fda07c..dc644caa91 100644 --- a/Makefile +++ b/Makefile @@ -130,12 +130,14 @@ release-notes: # needs to run as root to work test-e2e: $(GINKGO) - $(GINKGO) $(TESTFLAGS) \ - -r -p \ - --randomizeAllSpecs \ - --randomizeSuites \ - --slowSpecThreshold 60 \ - test + $(GINKGO) \ + -r \ + --randomize-all \ + --randomize-suites \ + --slow-spec-threshold 60s \ + test \ + -- \ + $(TESTFLAGS) test-crictl: $(GINKGO) # Run go test for templates_test.go and util_test.go diff --git a/test/e2e/events_test.go b/test/e2e/events_test.go index b0f25a2c3f..5c64283182 100644 --- a/test/e2e/events_test.go +++ b/test/e2e/events_test.go @@ -20,7 +20,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gbytes" - . "github.com/onsi/gomega/gexec" ) // The actual test suite @@ -40,20 +39,8 @@ var _ = t.Describe("events options validation", func() { // The actual test suite var _ = t.Describe("events", func() { - var ( - endpoint, testDir string - crio *Session - ) - BeforeEach(func() { - endpoint, testDir, crio = t.StartCrio() - }) - - AfterEach(func() { - t.StopCrio(testDir, crio) - }) - It("should succeed", func() { - session := t.CrictlWithEndpointNoWait(endpoint, "events") + session := t.CrictlNoWait("events") defer session.Terminate() Expect(session.Out).ToNot(Say("unknown method GetContainerEvents")) // no errors }) diff --git a/test/e2e/help_test.go b/test/e2e/help_test.go index 2a94f2b3ea..807ac838e5 100644 --- a/test/e2e/help_test.go +++ b/test/e2e/help_test.go @@ -18,7 +18,6 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega/gexec" ) // The actual test suite @@ -46,40 +45,27 @@ var _ = t.Describe("help", func() { // The actual test suite var _ = t.Describe("help subcommand", func() { - - var ( - endpoint, testDir string - crio *Session - ) - BeforeEach(func() { - endpoint, testDir, crio = t.StartCrio() - }) - - AfterEach(func() { - t.StopCrio(testDir, crio) - }) - It("should show help running rm without params", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, "rm", "crictl rm command") + t.CrictlExpectSuccess("rm", "crictl rm command") }) It("should show help running rmi without params", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, "rmi", "crictl rmi command") + t.CrictlExpectSuccess("rmi", "crictl rmi command") }) It("should show help running rmp without params", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, "rmp", "crictl rmp command") + t.CrictlExpectSuccess("rmp", "crictl rmp command") }) It("should not show help running rm -a", func() { - t.CrictlExpect(endpoint, "rm -a", 0, "", "No containers to remove") + t.CrictlExpect("rm -a", 0, "", "No containers to remove") }) It("should not show help running rmi -a", func() { - t.CrictlExpect(endpoint, "rmi -a", 0, "", "No images to remove") + t.CrictlExpect("rmi -a", 0, "", "No images to remove") }) It("should not show help running rmp -a", func() { - t.CrictlExpect(endpoint, "rmp -a", 0, "", "No pods to remove") + t.CrictlExpect("rmp -a", 0, "", "No pods to remove") }) }) diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go index 35049f33e8..5cc23e990f 100644 --- a/test/e2e/info_test.go +++ b/test/e2e/info_test.go @@ -18,26 +18,12 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega/gexec" ) // The actual test suite var _ = t.Describe("info", func() { - - var ( - endpoint, testDir string - crio *Session - ) - BeforeEach(func() { - endpoint, testDir, crio = t.StartCrio() - }) - - AfterEach(func() { - t.StopCrio(testDir, crio) - }) - It("should succeed", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, "info", "NetworkReady") + t.CrictlExpectSuccess("info", "NetworkReady") }) It("should fail with additional argument", func() { diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index b5e61381e9..4fa7051a6d 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -18,42 +18,35 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" ) // The actual test suite var _ = t.Describe("pull", func() { - var ( - endpoint, testDir string - crio *Session - ) - BeforeEach(func() { - endpoint, testDir, crio = t.StartCrio() - }) - - AfterEach(func() { - t.StopCrio(testDir, crio) - }) - const ( imageSuccessText = "Image is up to date" registry = "gcr.io/k8s-staging-cri-tools/" ) + AfterEach(func() { + Expect(t.Crictl("rmi -a")).To(Exit(0)) + }) + It("should succeed without tag or digest", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, + t.CrictlExpectSuccess( "pull "+registry+"test-image-1", imageSuccessText) }) It("should succeed with tag", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, + t.CrictlExpectSuccess( "pull "+registry+"test-image-1:latest", imageSuccessText) }) It("should succeed with digest", func() { - t.CrictlExpectSuccessWithEndpoint(endpoint, + t.CrictlExpectSuccess( "pull "+registry+"test-image-digest"+ "@sha256:9700f9a2f5bf2c45f2f605a0bd3bce7cf37420ec9d3ed50ac2758413308766bf", imageSuccessText) @@ -64,7 +57,6 @@ var _ = t.Describe("pull", func() { }) It("should fail on not existing image", func() { - t.CrictlExpectFailureWithEndpoint(endpoint, "pull localhost/wrong", - "", "pulling image") + t.CrictlExpectFailure("pull localhost/wrong", "", "pulling image") }) }) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 069b146f84..9c0f4b35fa 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -32,14 +32,9 @@ func TestE2E(t *testing.T) { var t *TestFramework -var _ = SynchronizedBeforeSuite(func() []byte { - // Setup only once - dir := SetupCrio() - return []byte(dir) - -}, func(dir []byte) { +var _ = BeforeSuite(func() { t = NewTestFramework() - t.Setup(string(dir)) + t.Setup() }) var _ = AfterSuite(func() { diff --git a/test/e2e/version_test.go b/test/e2e/version_test.go index cc463af082..1f0f417e49 100644 --- a/test/e2e/version_test.go +++ b/test/e2e/version_test.go @@ -23,13 +23,6 @@ import ( // The actual test suite var _ = t.Describe("version", func() { It("should succeed", func() { - // Given - endpoint, testDir, crio := t.StartCrio() - - // When - t.CrictlExpectSuccessWithEndpoint(endpoint, "version", "RuntimeName: cri-o") - - // Then - t.StopCrio(testDir, crio) + t.CrictlExpectSuccess("version", "RuntimeName: ") }) }) diff --git a/test/framework/framework.go b/test/framework/framework.go index d39818c759..963c8035ee 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -17,11 +17,9 @@ limitations under the License. package framework import ( - "errors" + "flag" "fmt" - "os" "os/exec" - "path/filepath" "strings" "time" @@ -32,25 +30,30 @@ import ( "github.com/sirupsen/logrus" ) -const timeout = 10 * time.Minute +var ( + crictlBinaryPath string + crictlRuntimeEndpoint string +) -// TestFramework is used to support commonly used test features -type TestFramework struct { - crioDir string +func init() { + flag.StringVar(&crictlBinaryPath, "crictl-binary-path", "", "`crictl` binary path to be used") + flag.StringVar(&crictlRuntimeEndpoint, "crictl-runtime-endpoint", "", "`crictl --runtime-endpoint` to be used") } +// TestFramework is used to support commonly used test features +type TestFramework struct{} + // NewTestFramework creates a new test framework instance func NewTestFramework() *TestFramework { - return &TestFramework{""} + return &TestFramework{} } // Setup is the global initialization function which runs before each test // suite -func (t *TestFramework) Setup(dir string) { +func (t *TestFramework) Setup() { // Global initialization for the whole framework goes in here logrus.SetLevel(logrus.DebugLevel) logrus.SetOutput(GinkgoWriter) - t.crioDir = dir } // Teardown is the global deinitialization function which runs after each test @@ -77,27 +80,41 @@ func cmd(workDir, format string, args ...interface{}) *Session { return session } +func crictlBinaryPathFlag() (path string) { + if crictlBinaryPath != "" { + return crictlBinaryPath + } + return "crictl" +} + +func crictlRuntimeEndpointFlag() string { + if crictlRuntimeEndpoint != "" { + return " --runtime-endpoint=" + crictlRuntimeEndpoint + } + return "" +} + // Convenience method for command creation in the current working directory func lcmd(format string, args ...interface{}) *Session { return cmd("", format, args...) } // Run crictl on the specified endpoint and return the resulting session -func (t *TestFramework) CrictlWithEndpoint(endpoint, args string) *Session { - return lcmd("crictl --runtime-endpoint=%s %s", endpoint, args).Wait(time.Minute) +func (t *TestFramework) Crictl(args string) *Session { + return lcmd("%s%s %s", crictlBinaryPathFlag(), crictlRuntimeEndpointFlag(), args).Wait(time.Minute) } // Run crictl on the specified endpoint and return the resulting session without wait -func (t *TestFramework) CrictlWithEndpointNoWait(endpoint, args string) *Session { - return lcmd("crictl --runtime-endpoint=%s %s", endpoint, args) +func (t *TestFramework) CrictlNoWait(args string) *Session { + return lcmd("%s%s %s", crictlBinaryPathFlag(), crictlRuntimeEndpointFlag(), args) } // Run crictl and expect exit, expectedOut, expectedErr func (t *TestFramework) CrictlExpect( - endpoint, args string, exit int, expectedOut, expectedErr string, + args string, exit int, expectedOut, expectedErr string, ) { // When - res := t.CrictlWithEndpoint(endpoint, args) + res := t.Crictl(args) // Then Expect(res).To(Exit(exit)) @@ -115,128 +132,12 @@ func (t *TestFramework) CrictlExpect( // Run crictl and expect success containing the specified output func (t *TestFramework) CrictlExpectSuccess(args, expectedOut string) { - t.CrictlExpect("", args, 0, expectedOut, "") -} - -// Run crictl and expect success containing the specified output -func (t *TestFramework) CrictlExpectSuccessWithEndpoint(endpoint, args, expectedOut string) { - t.CrictlExpect(endpoint, args, 0, expectedOut, "") + t.CrictlExpect(args, 0, expectedOut, "") } // Run crictl and expect error containing the specified outputs func (t *TestFramework) CrictlExpectFailure( args string, expectedOut, expectedErr string, ) { - t.CrictlExpect("", args, 1, expectedOut, expectedErr) -} - -// Run crictl and expect failure containing the specified output -func (t *TestFramework) CrictlExpectFailureWithEndpoint( - endpoint, args, expectedOut, expectedErr string, -) { - t.CrictlExpect(endpoint, args, 1, expectedOut, expectedErr) -} - -func SetupCrio() string { - const ( - crioURL = "https://github.com/cri-o/cri-o" - crioVersion = "v1.26.4" - conmonURL = "https://github.com/containers/conmon" - conmonVersion = "v2.1.7" - ) - tmpDir := filepath.Join(os.TempDir(), "crio-tmp") - - if _, err := os.Stat(tmpDir); errors.Is(err, os.ErrNotExist) { - logrus.Info("cloning and building CRI-O") - - Expect(checkoutAndBuild(tmpDir, crioURL, crioVersion)).To(BeNil()) - - conmonTmp := filepath.Join(tmpDir, "conmon") - checkoutAndBuild(conmonTmp, conmonURL, conmonVersion) - } - - return tmpDir -} - -func checkoutAndBuild(dir, url, rev string) error { - // A much faster approach than just cloning the whole repository - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - cmd(dir, "git init").Wait(timeout) - cmd(dir, "git remote add origin %s", url).Wait(timeout) - cmd(dir, "git fetch --depth=1 origin %s", rev).Wait(timeout) - cmd(dir, "git checkout -f FETCH_HEAD").Wait(timeout) - cmd(dir, "make").Wait(timeout) - return nil -} - -// Start the container runtime process -func (t *TestFramework) StartCrio() (string, string, *Session) { - // Create a new sandbox directory - tmpDir, err := os.MkdirTemp("", "crictl-e2e-") - Expect(err).To(BeNil()) - - // Copy everything together - lcmd("cp -R %s %s", filepath.Join(t.crioDir, "bin"), tmpDir).Wait() - - lcmd("cp %s %s", filepath.Join(t.crioDir, "test", "policy.json"), - tmpDir).Wait() - - for _, d := range []string{ - "cni-config", "root", "runroot", "log", "exits", "attach", - } { - Expect(os.MkdirAll(filepath.Join(tmpDir, d), 0755)).To(BeNil()) - } - - lcmd("cp %s %s", filepath.Join(t.crioDir, "contrib", "cni", - "10-crio-bridge.conf"), - filepath.Join(tmpDir, "cni-config")).Wait() - - endpoint := filepath.Join(tmpDir, "crio.sock") - - session := cmd(tmpDir, "%s"+ - " --config=%s"+ - " --listen=%s"+ - " --conmon=%s"+ - " --container-exits-dir=%s"+ - " --container-attach-socket-dir=%s"+ - " --log-dir=%s"+ - " --signature-policy=%s"+ - " --cni-config-dir=%s"+ - " --root=%s"+ - " --runroot=%s"+ - " --pinns-path=%s"+ - " --enable-pod-events", - filepath.Join(tmpDir, "bin", "crio"), - filepath.Join(t.crioDir, "crio.conf"), - endpoint, - filepath.Join(t.crioDir, "conmon", "bin", "conmon"), - filepath.Join(tmpDir, "exits"), - filepath.Join(tmpDir, "attach"), - filepath.Join(tmpDir, "log"), - filepath.Join(tmpDir, "policy.json"), - filepath.Join(tmpDir, "cni-config"), - filepath.Join(tmpDir, "root"), - filepath.Join(tmpDir, "runroot"), - filepath.Join(tmpDir, "bin", "pinns"), - ) - - endpoint = "unix://" + endpoint - - // Wait for the connection to be available - for i := 0; i < 100; i++ { - res := t.CrictlWithEndpoint(endpoint, "--timeout=200ms info") - if res.ExitCode() == 0 { - break - } - logrus.Info("Waiting for CRI-O to become ready") - time.Sleep(3 * time.Second) - } - return endpoint, tmpDir, session -} - -// Stop the container runtime process -func (t *TestFramework) StopCrio(testDir string, session *Session) { - Expect(session.Interrupt().Wait()).To(Exit(0)) + t.CrictlExpect(args, 1, expectedOut, expectedErr) }