Skip to content

Commit

Permalink
[test]Exclude features, add any feature check (microsoft#1853)
Browse files Browse the repository at this point in the history 
* test: Exclude features, add any feature check

Add `-exclude` flag to be able to specify all but a certain set of
flags to run, since it is common (especially locally) to want to run all
but a certain subset of tests.
eg, `functional.test.exe -exclude LCOWIntegrity` will run all test except
for those that require rego enforcement.

Add `require.AnyFeature` function to check that at least one of the listed
feature is specified.
This allows skipping a test that with subtests that individually require
non-overlapping features, which avoids running generalized test setup

Moved `"test/cri-containerd".requireBinary` to
`"test/pkg/require".Binary".

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

* PR: docs, comments, naming

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>

---------

Signed-off-by: Hamza El-Saawy <hamzaelsaawy@microsoft.com>
  • Loading branch information
@helsaawy @ambarve
helsaawy authored and ambarve committed Oct 17, 2023
1 parent 379efb1 commit 05a79ed
Show file tree
Hide file tree
Showing 20 changed files with 379 additions and 143 deletions.
3 changes: 1 addition & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,7 @@ jobs:
run: ${{ env.GOTESTCMD }} -gcflags=all=-d=checkptr -tags admin ./...

- name: Run non-functional tests
run: ${{ env.GOTESTCMD }} -mod=mod -gcflags=all=-d=checkptr -tags admin ./...

run: ${{ env.GOTESTCMD }} -mod=mod -gcflags=all=-d=checkptr ./internal/... ./pkg/...
working-directory: test

- name: Run containerd-shim-runhcs-v1 tests
Expand Down
2 changes: 2 additions & 0 deletions test/cri-containerd/container_gmsa_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

func Test_RunContainer_GMSA_WCOW(t *testing.T) {
requireFeatures(t, featureGMSA)
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor)

credSpec := gmsaSetup(t)
pullRequiredImages(t, []string{imageWindowsNanoserver})
Expand Down Expand Up @@ -98,6 +99,7 @@ func Test_RunContainer_GMSA_WCOW(t *testing.T) {
}
func Test_RunContainer_GMSA_Disabled(t *testing.T) {
requireFeatures(t, featureGMSA)
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor)

credSpec := "totally real and definitely not a fake or arbitrary gMSA credential spec that is 1000%% properly formatted as JSON"
pullRequiredImages(t, []string{imageWindowsNanoserver})
Expand Down
2 changes: 2 additions & 0 deletions test/cri-containerd/container_network_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ func Test_Container_Network_LCOW(t *testing.T) {
}

func Test_Container_Network_Hostname(t *testing.T) {
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)

type config struct {
name string
requiredFeatures []string
Expand Down
20 changes: 14 additions & 6 deletions test/cri-containerd/container_update_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ func calculateJobCPURate(hostProcs uint32, processorCount uint32) uint32 {
}

func Test_Container_UpdateResources_CPUShare(t *testing.T) {
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)
require.Build(t, osversion.V20H2)

type config struct {
name string
requiredFeatures []string
Expand Down Expand Up @@ -142,6 +144,9 @@ func Test_Container_UpdateResources_CPUShare(t *testing.T) {
}

func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) {
requireFeatures(t, featureCRIUpdateContainer)
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)

type config struct {
name string
requiredFeatures []string
Expand All @@ -153,23 +158,23 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) {
tests := []config{
{
name: "WCOW_Process",
requiredFeatures: []string{featureWCOWProcess, featureCRIUpdateContainer},
requiredFeatures: []string{featureWCOWProcess},
runtimeHandler: wcowProcessRuntimeHandler,
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"},
},
{
name: "WCOW_Hypervisor",
requiredFeatures: []string{featureWCOWHypervisor, featureCRIUpdateContainer},
requiredFeatures: []string{featureWCOWHypervisor},
runtimeHandler: wcowHypervisorRuntimeHandler,
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"},
},
{
name: "LCOW",
requiredFeatures: []string{featureLCOW, featureCRIUpdateContainer},
requiredFeatures: []string{featureLCOW},
runtimeHandler: lcowRuntimeHandler,
sandboxImage: imageLcowK8sPause,
containerImage: imageLcowAlpine,
Expand Down Expand Up @@ -249,6 +254,9 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) {
}

func Test_Container_UpdateResources_Memory(t *testing.T) {
requireFeatures(t, featureCRIUpdateContainer)
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)

type config struct {
name string
requiredFeatures []string
Expand All @@ -260,23 +268,23 @@ func Test_Container_UpdateResources_Memory(t *testing.T) {
tests := []config{
{
name: "WCOW_Process",
requiredFeatures: []string{featureWCOWProcess, featureCRIUpdateContainer},
requiredFeatures: []string{featureWCOWProcess},
runtimeHandler: wcowProcessRuntimeHandler,
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"},
},
{
name: "WCOW_Hypervisor",
requiredFeatures: []string{featureWCOWHypervisor, featureCRIUpdateContainer},
requiredFeatures: []string{featureWCOWHypervisor},
runtimeHandler: wcowHypervisorRuntimeHandler,
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"},
},
{
name: "LCOW",
requiredFeatures: []string{featureLCOW, featureCRIUpdateContainer},
requiredFeatures: []string{featureLCOW},
runtimeHandler: lcowRuntimeHandler,
sandboxImage: imageLcowK8sPause,
containerImage: imageLcowAlpine,
Expand Down
2 changes: 2 additions & 0 deletions test/cri-containerd/containerdrestart_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func Test_ContainerdRestart_LCOW(t *testing.T) {
// test restarting containers and pods
func Test_Container_CRI_Restart(t *testing.T) {
requireFeatures(t, featureCRIPlugin)
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)

client := newTestRuntimeClient(t)
pluginClient := newTestPluginClient(t)
Expand Down Expand Up @@ -225,6 +226,7 @@ func Test_Container_CRI_Restart_State(t *testing.T) {
wcowTestFile := `C:\Users\ContainerUser\t.txt`

requireFeatures(t, featureCRIPlugin)
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)

client := newTestRuntimeClient(t)
ctx, cancel := context.WithCancel(context.Background())
Expand Down
2 changes: 2 additions & 0 deletions test/cri-containerd/extended_task_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ func getPodProcessorInfo(ctx context.Context, podID string) (*extendedtask.Compu
}

func Test_ExtendedTask_ProcessorInfo(t *testing.T) {
requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW)

type config struct {
name string
requiredFeatures []string
Expand Down
153 changes: 70 additions & 83 deletions test/cri-containerd/jobcontainer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,9 +419,10 @@ func Test_RunContainer_HostVolumes_JobContainer_WCOW(t *testing.T) {
}

func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) {
client := newTestRuntimeClient(t)
requireFeatures(t, featureWCOWProcess, featureHostProcess)
require.ExactBuild(t, osversion.RS5)

client := newTestRuntimeClient(t)
dir := t.TempDir()

tmpfn := filepath.Join(dir, "tmpfile")
Expand Down Expand Up @@ -451,49 +452,43 @@ func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) {
}

type config struct {
name string
containerName string
requiredFeatures []string
sandboxImage string
containerImage string
exec []string
mounts []*runtime.Mount
name string
containerName string
sandboxImage string
containerImage string
exec []string
mounts []*runtime.Mount
}

tests := []config{
{
name: "JobContainer_VolumeMount_DriveLetter",
containerName: t.Name() + "-Container-DriveLetter",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
mounts: mountDriveLetter,
exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"},
name: "JobContainer_VolumeMount_DriveLetter",
containerName: t.Name() + "-Container-DriveLetter",
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
mounts: mountDriveLetter,
exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"},
},
{
name: "JobContainer_VolumeMount_NoDriveLetter",
containerName: t.Name() + "-Container-NoDriveLetter",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
mounts: mountNoDriveLetter,
exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"},
name: "JobContainer_VolumeMount_NoDriveLetter",
containerName: t.Name() + "-Container-NoDriveLetter",
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
mounts: mountNoDriveLetter,
exec: []string{"cmd", "/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\tmpfile"},
},
{
name: "JobContainer_VolumeMount_SingleFile",
containerName: t.Name() + "-Container-SingleFile",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
mounts: mountSingleFile,
exec: []string{"cmd", "/c", "type", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\testfile"},
name: "JobContainer_VolumeMount_SingleFile",
containerName: t.Name() + "-Container-SingleFile",
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
mounts: mountSingleFile,
exec: []string{"cmd", "/c", "type", "%CONTAINER_SANDBOX_MOUNT_POINT%\\path\\in\\container\\testfile"},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
requireFeatures(t, test.requiredFeatures...)

requiredImages := []string{test.sandboxImage, test.containerImage}
pullRequiredImages(t, requiredImages)

Expand Down Expand Up @@ -525,25 +520,25 @@ func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) {
}

func Test_RunContainer_JobContainer_Environment(t *testing.T) {
requireFeatures(t, featureWCOWProcess, featureHostProcess)

client := newTestRuntimeClient(t)

type config struct {
name string
containerName string
requiredFeatures []string
sandboxImage string
containerImage string
env []*runtime.KeyValue
exec []string
name string
containerName string
sandboxImage string
containerImage string
env []*runtime.KeyValue
exec []string
}

tests := []config{
{
name: "JobContainer_Env_NoMountPoint",
containerName: t.Name() + "-Container-WithNoMountPoint",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
name: "JobContainer_Env_NoMountPoint",
containerName: t.Name() + "-Container-WithNoMountPoint",
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
env: []*runtime.KeyValue{
{
Key: "PATH", Value: "C:\\Windows\\system32;C:\\Windows",
Expand All @@ -552,11 +547,10 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) {
exec: []string{"cmd", "/c", "IF", "%PATH%", "==", "C:\\Windows\\system32;C:\\Windows", "( exit 0 )", "ELSE", "(exit -1)"},
},
{
name: "JobContainer_VolumeMount_WithMountPoint",
containerName: t.Name() + "-Container-WithMountPoint",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
name: "JobContainer_VolumeMount_WithMountPoint",
containerName: t.Name() + "-Container-WithMountPoint",
sandboxImage: imageWindowsNanoserver,
containerImage: imageWindowsNanoserver,
env: []*runtime.KeyValue{
{
Key: "PATH", Value: "%CONTAINER_SANDBOX_MOUNT_POINT%\\apps\\vim\\;C:\\Windows\\system32;C:\\Windows",
Expand All @@ -568,8 +562,6 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) {

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
requireFeatures(t, test.requiredFeatures...)

requiredImages := []string{test.sandboxImage, test.containerImage}
pullRequiredImages(t, requiredImages)

Expand Down Expand Up @@ -602,57 +594,52 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) {
}

func Test_RunContainer_WorkingDirectory_JobContainer_WCOW(t *testing.T) {
client := newTestRuntimeClient(t)
requireFeatures(t, featureWCOWProcess, featureHostProcess)
require.ExactBuild(t, osversion.RS5)

client := newTestRuntimeClient(t)

type config struct {
name string
containerName string //nolint:unused // may be used in future tests
workDir string
requiredFeatures []string
sandboxImage string
containerImage string
cmd []string
name string
containerName string //nolint:unused // may be used in future tests
workDir string
sandboxImage string
containerImage string
cmd []string
}

tests := []config{
{
name: "JobContainer_WorkDir_DriveLetter",
workDir: "C:\\go\\",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"src\\workdir\\workdir.exe"},
name: "JobContainer_WorkDir_DriveLetter",
workDir: "C:\\go\\",
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"src\\workdir\\workdir.exe"},
},
{
name: "JobContainer_WorkDir_NoDriveLetter",
workDir: "/go",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"src/workdir/workdir.exe"},
name: "JobContainer_WorkDir_NoDriveLetter",
workDir: "/go",
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"src/workdir/workdir.exe"},
},
{
name: "JobContainer_WorkDir_Default", // Just use the workdir from the image, which is C:\\go\\src\\workdir
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"workdir.exe"},
name: "JobContainer_WorkDir_Default", // Just use the workdir from the image, which is C:\\go\\src\\workdir
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"workdir.exe"},
},
{
name: "JobContainer_WorkDir_EnvVar", // Test that putting the envvar in the workdir functions.
workDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\go\\src\\workdir\\",
requiredFeatures: []string{featureWCOWProcess, featureHostProcess},
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"workdir.exe"},
name: "JobContainer_WorkDir_EnvVar", // Test that putting the envvar in the workdir functions.
workDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\go\\src\\workdir\\",
sandboxImage: imageWindowsNanoserver,
containerImage: imageJobContainerWorkDir,
cmd: []string{"workdir.exe"},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
requireFeatures(t, test.requiredFeatures...)

requiredImages := []string{test.sandboxImage, test.containerImage}
pullRequiredImages(t, requiredImages)

Expand Down
Loading

0 comments on commit 05a79ed

Please sign in to comment.