From a0534fd393d7eff2e11110c25514023751d2641c Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Mon, 20 Jun 2022 15:16:50 -0500 Subject: [PATCH] Reduce smoke test suite size The smoke tests are taking too long, 5m on linux and 10m+ on windows on our CI servers. The goal for the smoke tests is <3m so that it's easy to run and vet that you didn't completely break porter. I've moved some of the smoke tests into the integration test suite so that we are only focusing on the most important checks in the smoke test suite. Signed-off-by: Carolyn Van Slyck --- tests/integration/archive_test.go | 49 ++++++++++++++++++- tests/{smoke => integration}/copy_test.go | 12 ++--- tests/integration/telemetry_test.go | 42 ++++++++++++++++ tests/smoke/archive_test.go | 59 ----------------------- tests/smoke/desiredstate_test.go | 1 - tests/smoke/hello_test.go | 1 - tests/smoke/main_test.go | 1 - tests/smoke/telemetry_test.go | 53 -------------------- 8 files changed, 92 insertions(+), 126 deletions(-) rename tests/{smoke => integration}/copy_test.go (83%) delete mode 100644 tests/smoke/archive_test.go delete mode 100644 tests/smoke/telemetry_test.go diff --git a/tests/integration/archive_test.go b/tests/integration/archive_test.go index 45249a28c..6850747b7 100644 --- a/tests/integration/archive_test.go +++ b/tests/integration/archive_test.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package integration @@ -14,7 +13,12 @@ import ( "get.porter.sh/porter/pkg" "get.porter.sh/porter/pkg/porter" + "get.porter.sh/porter/pkg/yaml" "get.porter.sh/porter/tests" + "get.porter.sh/porter/tests/testdata" + "get.porter.sh/porter/tests/tester" + "github.com/carolynvs/magex/mgx" + "github.com/carolynvs/magex/shx" "github.com/cnabio/cnab-go/bundle/loader" "github.com/cnabio/cnab-go/packager" "github.com/cnabio/cnab-to-oci/relocation" @@ -22,7 +26,48 @@ import ( "github.com/stretchr/testify/require" ) -func TestArchive(t *testing.T) { +// Validate that when we archive a bundle, everything is included inside +// We should be able to archive, move to a disconnected environment +// and publish again without accidentally referencing artifacts from the original environment. +func TestArchive_AirgappedEnvironment(t *testing.T) { + test, err := tester.NewTest(t) + defer test.Close() + require.NoError(t, err, "test setup failed") + + // Start a temp registry + tempRegistryId, err := shx.OutputE("docker", "run", "-d", "-P", "registry:2") + require.NoError(t, err, "Could not start a temporary registry") + stopTempRegistry := func() error { + return shx.RunE("docker", "rm", "-vf", tempRegistryId) + } + defer stopTempRegistry() + + // Get the port that it is running on + tempRegistryPort, err := shx.OutputE("docker", "inspect", tempRegistryId, "--format", `{{ (index (index .NetworkSettings.Ports "5000/tcp") 0).HostPort }}`) + require.NoError(t, err, "Could not get the published port of the temporary registry") + + // Publish referenced bundle to one location + localRegRef := fmt.Sprintf("localhost:%s/whalesayd:latest", tempRegistryPort) + require.NoError(t, shx.RunE("docker", "pull", "carolynvs/whalesayd@sha256:8b92b7269f59e3ed824e811a1ff1ee64f0d44c0218efefada57a4bebc2d7ef6f")) + require.NoError(t, shx.RunE("docker", "tag", "carolynvs/whalesayd@sha256:8b92b7269f59e3ed824e811a1ff1ee64f0d44c0218efefada57a4bebc2d7ef6f", localRegRef)) + require.NoError(t, shx.RunE("docker", "push", localRegRef)) + + // publish a test bundle that reference the image from the temp registry + originTestBun := filepath.Join(test.RepoRoot, fmt.Sprintf("tests/testdata/%s/porter.yaml", testdata.MyBunsWithImgReference)) + testBun := filepath.Join(test.TestDir, "mybuns-img-reference.yaml") + mgx.Must(shx.Copy(originTestBun, testBun)) + test.EditYaml(testBun, func(yq *yaml.Editor) error { + return yq.SetValue("images.whalesayd.repository", fmt.Sprintf("localhost:%s/whalesayd", tempRegistryPort)) + }) + test.RequirePorter("publish", "--file", "mybuns-img-reference.yaml", "--dir", test.TestDir) + stopTempRegistry() + + archiveFilePath := filepath.Join(test.TestDir, "archive-test.tgz") + test.RequirePorter("archive", archiveFilePath, "--reference", testdata.MyBunsWithImgReferenceRef) +} + +// Validate that archiving a bundle twice results in the same digest +func TestArchive_StableDigest(t *testing.T) { t.Parallel() p := porter.NewTestPorter(t) diff --git a/tests/smoke/copy_test.go b/tests/integration/copy_test.go similarity index 83% rename from tests/smoke/copy_test.go rename to tests/integration/copy_test.go index 2920bc10e..c7080d7d2 100644 --- a/tests/smoke/copy_test.go +++ b/tests/integration/copy_test.go @@ -1,7 +1,6 @@ -//go:build smoke -// +build smoke +//go:build integration -package smoke +package integration import ( "encoding/json" @@ -14,12 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -// Start up another docker registry to host the original bundle -// Publish a bundle to the temporary registry -// Copy the bundle to our integration test registry -// Stop the temporary registry -// Copy the bundle to another location, this will fail unless we are properly using the relocation map -func TestCopy(t *testing.T) { +func TestCopy_UsesRelocationMap(t *testing.T) { test, err := tester.NewTest(t) defer test.Close() require.NoError(t, err, "test setup failed") diff --git a/tests/integration/telemetry_test.go b/tests/integration/telemetry_test.go index 92c7e2ab4..7c4532cf8 100644 --- a/tests/integration/telemetry_test.go +++ b/tests/integration/telemetry_test.go @@ -4,12 +4,17 @@ package integration import ( "context" + "fmt" + "io/ioutil" "os" + "path/filepath" "testing" "time" "get.porter.sh/porter/pkg/porter" + "get.porter.sh/porter/tests" "get.porter.sh/porter/tests/tester" + "github.com/carolynvs/magex/shx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,3 +53,40 @@ func TestTelemetrySetup(t *testing.T) { defer log.Close() assert.True(t, log.IsTracingEnabled()) } + +// Test that telemetry data is being exported both from porter and the plugins +func TestTelemetry_IncludesPluginLogs(t *testing.T) { + // I am always using require, so that we stop immediately upon an error + // A long test is hard to debug when it fails in the middle and keeps going + test, err := tester.NewTestWithConfig(t, "tests/testdata/config/config-with-telemetry.yaml") + defer test.Close() + require.NoError(t, err, "test setup failed") + + // Enable telemetry + err = shx.Copy(filepath.Join(test.RepoRoot, "tests/testdata/config/config-with-telemetry.yaml"), filepath.Join(test.PorterHomeDir, "config.yaml")) + require.NoError(t, err, "error copying config file into PORTER_HOME") + + // Make a call that will call a plugin + _, output, err := test.RunPorter("list") + fmt.Println(output) + require.NoError(t, err, "porter list failed") + + // Read the traces generated for that call + tracesDir := filepath.Join(test.PorterHomeDir, "traces") + traces, err := os.ReadDir(tracesDir) + require.NoError(t, err, "error getting a list of the traces directory in PORTER_HOME") + require.Len(t, traces, 2, "expected 2 trace files to be exported") + + // Validate we have trace data for porter (files are returned in descending order, which is why we know which to read first) + porterTraceName := filepath.Join(tracesDir, traces[1].Name()) + porterTrace, err := ioutil.ReadFile(porterTraceName) + require.NoError(t, err, "error reading porter's trace file %s", porterTraceName) + tests.RequireOutputContains(t, string(porterTrace), `{"Key":"service.name","Value":{"Type":"STRING","Value":"porter"}}`, "no spans for porter were exported") + + // Validate we have trace data for porter + pluginTraceName := filepath.Join(tracesDir, traces[0].Name()) + require.Contains(t, pluginTraceName, "storage.porter.mongodb", "expected the plugin trace to be for the mongodb plugin") + pluginTrace, err := ioutil.ReadFile(pluginTraceName) + require.NoError(t, err, "error reading the plugin's trace file %s", pluginTraceName) + tests.RequireOutputContains(t, string(pluginTrace), `{"Key":"service.name","Value":{"Type":"STRING","Value":"storage.porter.mongodb"}}`, "no spans for the plugins were exported") +} diff --git a/tests/smoke/archive_test.go b/tests/smoke/archive_test.go deleted file mode 100644 index 579a170c2..000000000 --- a/tests/smoke/archive_test.go +++ /dev/null @@ -1,59 +0,0 @@ -//go:build smoke -// +build smoke - -package smoke - -import ( - "fmt" - "path/filepath" - "testing" - - "get.porter.sh/porter/pkg/yaml" - "get.porter.sh/porter/tests/testdata" - "get.porter.sh/porter/tests/tester" - "github.com/carolynvs/magex/mgx" - "github.com/carolynvs/magex/shx" - "github.com/stretchr/testify/require" -) - -// Start up another docker registry to host the original bundle -// Publish a bundle to the temporary registry -// Archive the bundle from the temporary registry -// Stop the temporary registry -// Publish the bundle from the archive file -func TestArchive(t *testing.T) { - test, err := tester.NewTest(t) - defer test.Close() - require.NoError(t, err, "test setup failed") - - // Start a temp registry - tempRegistryId, err := shx.OutputE("docker", "run", "-d", "-P", "registry:2") - require.NoError(t, err, "Could not start a temporary registry") - stopTempRegistry := func() error { - return shx.RunE("docker", "rm", "-vf", tempRegistryId) - } - defer stopTempRegistry() - - // Get the port that it is running on - tempRegistryPort, err := shx.OutputE("docker", "inspect", tempRegistryId, "--format", `{{ (index (index .NetworkSettings.Ports "5000/tcp") 0).HostPort }}`) - require.NoError(t, err, "Could not get the published port of the temporary registry") - - // Publish referenced bundle to one location - localRegRef := fmt.Sprintf("localhost:%s/whalesayd:latest", tempRegistryPort) - require.NoError(t, shx.RunE("docker", "pull", "carolynvs/whalesayd@sha256:8b92b7269f59e3ed824e811a1ff1ee64f0d44c0218efefada57a4bebc2d7ef6f")) - require.NoError(t, shx.RunE("docker", "tag", "carolynvs/whalesayd@sha256:8b92b7269f59e3ed824e811a1ff1ee64f0d44c0218efefada57a4bebc2d7ef6f", localRegRef)) - require.NoError(t, shx.RunE("docker", "push", localRegRef)) - - // publish a test bundle that reference the image from the temp registry - originTestBun := filepath.Join(test.RepoRoot, fmt.Sprintf("tests/testdata/%s/porter.yaml", testdata.MyBunsWithImgReference)) - testBun := filepath.Join(test.TestDir, "mybuns-img-reference.yaml") - mgx.Must(shx.Copy(originTestBun, testBun)) - test.EditYaml(testBun, func(yq *yaml.Editor) error { - return yq.SetValue("images.whalesayd.repository", fmt.Sprintf("localhost:%s/whalesayd", tempRegistryPort)) - }) - test.RequirePorter("publish", "--file", "mybuns-img-reference.yaml", "--dir", test.TestDir) - stopTempRegistry() - - archiveFilePath := filepath.Join(test.TestDir, "archive-test.tgz") - test.RequirePorter("archive", archiveFilePath, "--reference", testdata.MyBunsWithImgReferenceRef) -} diff --git a/tests/smoke/desiredstate_test.go b/tests/smoke/desiredstate_test.go index 4dec72d0b..6adc21f56 100644 --- a/tests/smoke/desiredstate_test.go +++ b/tests/smoke/desiredstate_test.go @@ -1,5 +1,4 @@ //go:build smoke -// +build smoke package smoke diff --git a/tests/smoke/hello_test.go b/tests/smoke/hello_test.go index 77c2acf0f..38680d907 100644 --- a/tests/smoke/hello_test.go +++ b/tests/smoke/hello_test.go @@ -1,5 +1,4 @@ //go:build smoke -// +build smoke package smoke diff --git a/tests/smoke/main_test.go b/tests/smoke/main_test.go index 1246d0809..9204067ee 100644 --- a/tests/smoke/main_test.go +++ b/tests/smoke/main_test.go @@ -1,5 +1,4 @@ //go:build smoke -// +build smoke package smoke diff --git a/tests/smoke/telemetry_test.go b/tests/smoke/telemetry_test.go deleted file mode 100644 index 0e9659af2..000000000 --- a/tests/smoke/telemetry_test.go +++ /dev/null @@ -1,53 +0,0 @@ -//go:build smoke - -package smoke - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "get.porter.sh/porter/tests" - "get.porter.sh/porter/tests/tester" - "github.com/carolynvs/magex/shx" - "github.com/stretchr/testify/require" -) - -// Test that telemetry data is being exported -func TestTelemetry(t *testing.T) { - // I am always using require, so that we stop immediately upon an error - // A long test is hard to debug when it fails in the middle and keeps going - test, err := tester.NewTestWithConfig(t, "tests/testdata/config/config-with-telemetry.yaml") - defer test.Close() - require.NoError(t, err, "test setup failed") - - // Enable telemetry - err = shx.Copy(filepath.Join(test.RepoRoot, "tests/testdata/config/config-with-telemetry.yaml"), filepath.Join(test.PorterHomeDir, "config.yaml")) - require.NoError(t, err, "error copying config file into PORTER_HOME") - - // Make a call that will call a plugin - _, output, err := test.RunPorter("list") - fmt.Println(output) - require.NoError(t, err, "porter list failed") - - // Read the traces generated for that call - tracesDir := filepath.Join(test.PorterHomeDir, "traces") - traces, err := os.ReadDir(tracesDir) - require.NoError(t, err, "error getting a list of the traces directory in PORTER_HOME") - require.Len(t, traces, 2, "expected 2 trace files to be exported") - - // Validate we have trace data for porter (files are returned in descending order, which is why we know which to read first) - porterTraceName := filepath.Join(tracesDir, traces[1].Name()) - porterTrace, err := ioutil.ReadFile(porterTraceName) - require.NoError(t, err, "error reading porter's trace file %s", porterTraceName) - tests.RequireOutputContains(t, string(porterTrace), `{"Key":"service.name","Value":{"Type":"STRING","Value":"porter"}}`, "no spans for porter were exported") - - // Validate we have trace data for porter - pluginTraceName := filepath.Join(tracesDir, traces[0].Name()) - require.Contains(t, pluginTraceName, "storage.porter.mongodb", "expected the plugin trace to be for the mongodb plugin") - pluginTrace, err := ioutil.ReadFile(pluginTraceName) - require.NoError(t, err, "error reading the plugin's trace file %s", pluginTraceName) - tests.RequireOutputContains(t, string(pluginTrace), `{"Key":"service.name","Value":{"Type":"STRING","Value":"storage.porter.mongodb"}}`, "no spans for the plugins were exported") -}