no team software gitops (#20847)

#20464 

Adding gitops support for a top level `software` key to be used to
manage installable software into "no team".

- [ ] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [X] Added/updated tests
- [X] Manual QA for all new/changed functionality

---------

Co-authored-by: Victor Lyuboslavsky <victor.lyuboslavsky@gmail.com>

cmd/fleetctl/apply.go

@@ -90,7 +90,7 @@ func applyCommand() *cli.Command {
 				opts.TeamForPolicies = policiesTeamName
 			}
 			baseDir := filepath.Dir(flFilename)
-			_, err = fleetClient.ApplyGroup(c.Context, specs, baseDir, logf, opts)
+			_, err = fleetClient.ApplyGroup(c.Context, specs, baseDir, logf, nil, opts)
 			if err != nil {
 				return err
 			}

cmd/fleetctl/gitops.go

@@ -90,7 +90,7 @@ func gitopsCommand() *cli.Command {
 			secrets := make(map[string]struct{})
 			for _, flFilename := range flFilenames.Value() {
 				baseDir := filepath.Dir(flFilename)
-				config, err := spec.GitOpsFromFile(flFilename, baseDir)
+				config, err := spec.GitOpsFromFile(flFilename, baseDir, appConfig)
 				if err != nil {
 					return err
 				}

cmd/fleetctl/gitops_enterprise_integration_test.go

@@ -176,6 +176,7 @@ contexts:
 		fmt.Sprintf(
 			`
 controls:
+software:
 queries:
 policies:
 agent_options:
@@ -230,5 +231,4 @@ team_settings:
 	for _, fileName := range teamFileNames {
 		_ = runAppForTest(t, []string{"gitops", "--config", fleetctlConfig.Name(), "-f", fileName})
 	}
-
 }

cmd/fleetctl/gitops_test.go

@@ -36,7 +36,7 @@ const (
 	orgName        = "GitOps Test"
 )
 
-func TestFilenameValidation(t *testing.T) {
+func TestFilenameGitOpsValidation(t *testing.T) {
 	filename := strings.Repeat("a", filenameMaxLength+1)
 	_, err := runAppNoChecks([]string{"gitops", "-f", filename})
 	assert.ErrorContains(t, err, "file name must be less than")
@@ -207,6 +207,9 @@ func TestBasicGlobalPremiumGitOps(t *testing.T) {
 	ds.NewJobFunc = func(ctx context.Context, job *fleet.Job) (*fleet.Job, error) {
 		return &fleet.Job{}, nil
 	}
+	ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error {
+		return nil
+	}
 
 	tmpFile, err := os.CreateTemp(t.TempDir(), "*.yml")
 	require.NoError(t, err)
@@ -238,6 +241,7 @@ org_settings:
     org_logo_url_light_background: ""
     org_name: ${ORG_NAME}
   secrets:
+software:
 `,
 	)
 	require.NoError(t, err)
@@ -381,6 +385,7 @@ agent_options:
 name: ${TEST_TEAM_NAME}
 team_settings:
   secrets: ${TEST_SECRET}
+software:
 `,
 	)
 	require.NoError(t, err)
@@ -538,6 +543,7 @@ func TestFullGlobalGitOps(t *testing.T) {
 	t.Setenv("FLEET_SERVER_URL", fleetServerURL)
 	t.Setenv("ORG_NAME", orgName)
 	t.Setenv("APPLE_BM_DEFAULT_TEAM", teamName)
+	t.Setenv("SOFTWARE_INSTALLER_URL", fleetServerURL)
 	file := "./testdata/gitops/global_config_no_paths.yml"
 
 	// Dry run should fail because Apple BM Default Team does not exist and premium license is not set
@@ -834,6 +840,7 @@ agent_options:
 name: ${TEST_TEAM_NAME}
 team_settings:
   secrets: [{"secret":"${TEST_SECRET}"}]
+software:
 `,
 	)
 	require.NoError(t, err)
@@ -1011,6 +1018,7 @@ org_settings:
     org_logo_url_light_background: ""
     org_name: ${ORG_NAME}
   secrets: [{"secret":"globalSecret"}]
+software:
 `,
 	)
 	require.NoError(t, err)
@@ -1030,6 +1038,7 @@ agent_options:
 name: ${TEST_TEAM_NAME}
 team_settings:
   secrets: [{"secret":"${TEST_SECRET}"}]
+software:
 `,
 	)
 	require.NoError(t, err)
@@ -1045,6 +1054,7 @@ agent_options:
 name: ${TEST_TEAM_NAME}
 team_settings:
   secrets: [{"secret":"${TEST_SECRET}"},{"secret":"globalSecret"}]
+software:
 `,
 	)
 	require.NoError(t, err)
@@ -1222,7 +1232,7 @@ func TestTeamSofwareInstallersGitOps(t *testing.T) {
 		{"testdata/gitops/team_software_installer_install_not_found.yml", "no such file or directory"},
 		{"testdata/gitops/team_software_installer_post_install_not_found.yml", "no such file or directory"},
 		{"testdata/gitops/team_software_installer_no_url.yml", "software URL is required"},
-		{"testdata/gitops/team_software_installer_invalid_self_service_value.yml", "cannot unmarshal string into Go struct field TeamSpecSoftware.packages of type bool"},
+		{"testdata/gitops/team_software_installer_invalid_self_service_value.yml", "cannot unmarshal string into Go struct field SoftwareSpec.packages of type bool"},
 	}
 	for _, c := range cases {
 		t.Run(filepath.Base(c.file), func(t *testing.T) {
@@ -1255,6 +1265,39 @@ func TestTeamSoftwareInstallersGitopsQueryEnv(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestNoTeamSoftwareInstallersGitOps(t *testing.T) {
+	startSoftwareInstallerServer(t)
+
+	cases := []struct {
+		file    string
+		wantErr string
+	}{
+		{"testdata/gitops/no_team_software_installer_not_found.yml", "Please make sure that URLs are publicy accessible to the internet."},
+		{"testdata/gitops/no_team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe or .deb."},
+		{"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 500 MB"},
+		{"testdata/gitops/no_team_software_installer_valid.yml", ""},
+		{"testdata/gitops/no_team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."},
+		{"testdata/gitops/no_team_software_installer_pre_condition_not_found.yml", "no such file or directory"},
+		{"testdata/gitops/no_team_software_installer_install_not_found.yml", "no such file or directory"},
+		{"testdata/gitops/no_team_software_installer_post_install_not_found.yml", "no such file or directory"},
+		{"testdata/gitops/no_team_software_installer_no_url.yml", "software URL is required"},
+		{"testdata/gitops/no_team_software_installer_invalid_self_service_value.yml", "cannot unmarshal string into Go struct field SoftwareSpec.packages of type bool"},
+	}
+	for _, c := range cases {
+		t.Run(filepath.Base(c.file), func(t *testing.T) {
+			setupFullGitOpsPremiumServer(t)
+
+			t.Setenv("APPLE_BM_DEFAULT_TEAM", "")
+			_, err := runAppNoChecks([]string{"gitops", "-f", c.file})
+			if c.wantErr == "" {
+				require.NoError(t, err)
+			} else {
+				require.ErrorContains(t, err, c.wantErr)
+			}
+		})
+	}
+}
+
 func TestTeamVPPAppsGitOps(t *testing.T) {
 	config := &appleVPPConfigSrvConf{
 		Assets: []vpp.Asset{

cmd/fleetctl/preview.go

@@ -387,7 +387,7 @@ Use the stop and reset subcommands to manage the server and dependencies once st
 			}
 			// this only applies standard queries, the base directory is not used,
 			// so pass in the current working directory.
-			_, err = client.ApplyGroup(c.Context, specs, ".", logf, fleet.ApplyClientSpecOptions{})
+			_, err = client.ApplyGroup(c.Context, specs, ".", logf, nil, fleet.ApplyClientSpecOptions{})
 			if err != nil {
 				return err
 			}

cmd/fleetctl/testdata/gitops/global_config_no_paths.yml

@@ -187,3 +187,4 @@ org_settings:
   secrets: # These secrets are used to enroll hosts to the "All teams" team
     - secret: SampleSecret123
     - secret: ABC
+software:

cmd/fleetctl/testdata/gitops/global_macos_custom_settings_valid_deprecated.yml

@@ -91,3 +91,4 @@ org_settings:
     databases_path: ""
   secrets:
     - secret: ABC
+software:

cmd/fleetctl/testdata/gitops/global_macos_windows_custom_settings_valid.yml

@@ -97,3 +97,4 @@ org_settings:
     databases_path: ""
   secrets: 
     - secret: ABC
+software:

cmd/fleetctl/testdata/gitops/global_windows_custom_settings_invalid_label_mix.yml

@@ -93,3 +93,4 @@ org_settings:
     databases_path: ""
   secrets:
     - secret: ABC
+software:

cmd/fleetctl/testdata/gitops/global_windows_custom_settings_unknown_label.yml

@@ -91,3 +91,4 @@ org_settings:
     databases_path: ""
   secrets:
     - secret: ABC
+software:

cmd/fleetctl/testdata/gitops/no_team_software_installer_install_not_found.yml

@@ -0,0 +1,19 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb
+      install_script:
+        path: lib/notfound.sh

cmd/fleetctl/testdata/gitops/no_team_software_installer_invalid_self_service_value.yml

@@ -0,0 +1,18 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/invalidtype.txt
+      self_service: "not a boolean"

cmd/fleetctl/testdata/gitops/no_team_software_installer_no_url.yml

@@ -0,0 +1,22 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - install_script:
+        path: lib/install_ruby.sh
+      pre_install_query:
+        path: lib/query_ruby.yml
+      post_install_script:
+        path: lib/post_install_ruby.sh

cmd/fleetctl/testdata/gitops/no_team_software_installer_not_found.yml

@@ -0,0 +1,17 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/notfound.deb

cmd/fleetctl/testdata/gitops/no_team_software_installer_post_install_not_found.yml

@@ -0,0 +1,21 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb
+      install_script:
+        path: lib/install_ruby.sh
+      post_install_script:
+        path: lib/notfound.sh

cmd/fleetctl/testdata/gitops/no_team_software_installer_pre_condition_multiple_queries.yml

@@ -0,0 +1,23 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb
+      install_script:
+        path: lib/install_ruby.sh
+      pre_install_query:
+        path: lib/query_multiple.yml
+      post_install_script:
+        path: lib/post_install_ruby.sh

cmd/fleetctl/testdata/gitops/no_team_software_installer_pre_condition_not_found.yml

@@ -0,0 +1,21 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/ruby.deb
+      install_script:
+        path: lib/install_ruby.sh
+      pre_install_query:
+        path: lib/notfound.yml

cmd/fleetctl/testdata/gitops/no_team_software_installer_too_large.yml

@@ -0,0 +1,17 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/toolarge.deb

cmd/fleetctl/testdata/gitops/no_team_software_installer_unsupported.yml

@@ -0,0 +1,17 @@
+# Test config
+controls:
+queries:
+policies:
+agent_options:
+org_settings:
+  server_settings:
+    server_url: $FLEET_SERVER_URL
+  org_info:
+    contact_url: https://example.com/contact
+    org_logo_url: ""
+    org_logo_url_light_background: ""
+    org_name: ${ORG_NAME}
+  secrets: [{"secret":"globalSecret"}]
+software:
+  packages:
+    - url: ${SOFTWARE_INSTALLER_URL}/invalidtype.txt