From 861d9aff147e97f0b20fc5eb532ae6f7589a6b04 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Tue, 2 Jul 2024 15:52:08 -0400 Subject: [PATCH] OCPBUGS-36344: Add CIP relevant mirrors to sigstore attachement cfg Add icsp/idms/itms mirrors of CIP scope to /etc/containers/registries.d, so sigstore attachment will be used during the image pull and verification. Signed-off-by: Qi Wang --- .../container_runtime_config_controller.go | 2 +- ...ontainer_runtime_config_controller_test.go | 53 +++++- .../container-runtime-config/helpers.go | 134 +++++++++++++- .../container-runtime-config/helpers_test.go | 167 ++++++++++++++++-- 4 files changed, 335 insertions(+), 21 deletions(-) diff --git a/pkg/controller/container-runtime-config/container_runtime_config_controller.go b/pkg/controller/container-runtime-config/container_runtime_config_controller.go index de9832c163..132ede057d 100644 --- a/pkg/controller/container-runtime-config/container_runtime_config_controller.go +++ b/pkg/controller/container-runtime-config/container_runtime_config_controller.go @@ -1007,7 +1007,7 @@ func registriesConfigIgnition(templateDir string, controllerConfig *mcfgv1.Contr return nil, fmt.Errorf("could not update policy json with new changes: %w", err) } // generates configuration under /etc/containers/registries.d to enable sigstore verification - sigstoreRegistriesConfigYaml, err = generateSigstoreRegistriesdConfig(clusterScopePolicies) + sigstoreRegistriesConfigYaml, err = generateSigstoreRegistriesdConfig(clusterScopePolicies, registriesTOML) if err != nil { return nil, err } diff --git a/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go b/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go index 21a1641d2c..f81414d18a 100644 --- a/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go +++ b/pkg/controller/container-runtime-config/container_runtime_config_controller_test.go @@ -534,7 +534,7 @@ func verifyRegistriesConfigAndPolicyJSONContents(t *testing.T, mc *mcfgv1.Machin } if verifyImagePoliciesRegistriesConfig { - expectedRegistriesConfd, err := generateSigstoreRegistriesdConfig(clusterScopePolicies) + expectedRegistriesConfd, err := generateSigstoreRegistriesdConfig(clusterScopePolicies, expectedRegistriesConf) require.NoError(t, err) foundFile := false @@ -1794,3 +1794,54 @@ func TestClusterImagePolicyCreate(t *testing.T) { }) } } + +func TestSigstoreRegistriesConfigIDMSandCIPCreate(t *testing.T) { + for _, platform := range []apicfgv1.PlatformType{apicfgv1.AWSPlatformType, apicfgv1.NonePlatformType, "unrecognized"} { + t.Run(string(platform), func(t *testing.T) { + f := newFixture(t) + + cc := newControllerConfig(ctrlcommon.ControllerConfigName, platform) + mcp := helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0") + mcp2 := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0") + imgcfg1 := newImageConfig("cluster", &apicfgv1.RegistrySources{InsecureRegistries: []string{"blah.io"}, AllowedRegistries: []string{"example.com"}, ContainerRuntimeSearchRegistries: []string{"search-reg.io"}}) + + cvcfg1 := newClusterVersionConfig("version", "test.io/myuser/myimage:test") + keyReg1, _ := getManagedKeyReg(mcp, nil) + keyReg2, _ := getManagedKeyReg(mcp2, nil) + + mcs1 := helpers.NewMachineConfig(keyReg1, map[string]string{"node-role": "master"}, "dummy://", []ign3types.File{{}}) + mcs2 := helpers.NewMachineConfig(keyReg2, map[string]string{"node-role": "worker"}, "dummy://", []ign3types.File{{}}) + + // idms source is the same as cip scope + idms := newIDMS("built-in", []apicfgv1.ImageDigestMirrors{ + {Source: "built-in-source.example.com", Mirrors: []apicfgv1.ImageMirror{"built-in-mirror.example.com"}}, + }) + clusterimgPolicy := newClusterImagePolicyWithPublicKey("built-in-source.example.com", []string{"example.com"}, []byte("foo bar")) + f.ccLister = append(f.ccLister, cc) + f.mcpLister = append(f.mcpLister, mcp) + f.mcpLister = append(f.mcpLister, mcp2) + f.imgLister = append(f.imgLister, imgcfg1) + f.idmsLister = append(f.idmsLister, idms) + f.clusterImagePolicyLister = append(f.clusterImagePolicyLister, clusterimgPolicy) + f.cvLister = append(f.cvLister, cvcfg1) + f.imgObjects = append(f.imgObjects, imgcfg1) + + f.expectGetMachineConfigAction(mcs1) + f.expectGetMachineConfigAction(mcs1) + f.expectGetMachineConfigAction(mcs1) + f.expectCreateMachineConfigAction(mcs1) + + f.expectGetMachineConfigAction(mcs2) + f.expectGetMachineConfigAction(mcs2) + f.expectGetMachineConfigAction(mcs2) + + f.expectCreateMachineConfigAction(mcs2) + + f.run("") + + for _, mcName := range []string{mcs1.Name, mcs2.Name} { + f.verifyRegistriesConfigAndPolicyJSONContents(t, mcName, imgcfg1, nil, idms, nil, clusterimgPolicy, cc.Spec.ReleaseImage, true, true, true) + } + }) + } +} diff --git a/pkg/controller/container-runtime-config/helpers.go b/pkg/controller/container-runtime-config/helpers.go index 1d11282629..ab6bcfdc7b 100644 --- a/pkg/controller/container-runtime-config/helpers.go +++ b/pkg/controller/container-runtime-config/helpers.go @@ -20,6 +20,7 @@ import ( storageconfig "github.com/containers/storage/pkg/config" ign3types "github.com/coreos/ignition/v2/config/v3_4/types" "github.com/ghodss/yaml" + "github.com/opencontainers/go-digest" apicfgv1 "github.com/openshift/api/config/v1" apicfgv1alpha1 "github.com/openshift/api/config/v1alpha1" apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1" @@ -933,17 +934,73 @@ func validateClusterImagePolicyWithAllowedBlockedRegistries(clusterScopePolicies return nil } -func generateSigstoreRegistriesdConfig(clusterScopePolicies map[string]signature.PolicyRequirements) ([]byte, error) { +func generateSigstoreRegistriesdConfig(clusterScopePolicies map[string]signature.PolicyRequirements, registriesTOML []byte) ([]byte, error) { if len(clusterScopePolicies) == 0 { return nil, nil } - registriesDockerConfig := make(map[string]dockerConfig) - sigstoreAttachment := dockerConfig{ - UseSigstoreAttachments: true, + var ( + err error + tmpFile *os.File + registriesConf = sysregistriesv2.V2RegistriesConf{} + registriesDockerConfig = make(map[string]dockerConfig) + sigstoreAttachment = dockerConfig{ + UseSigstoreAttachments: true, + } + ) + + if registriesTOML != nil { + tmpFile, err = os.CreateTemp("", "regtemp") + if err != nil { + return nil, err + } + defer func() { + tmpFile.Close() + os.Remove(tmpFile.Name()) + }() + + _, err = tmpFile.Write(registriesTOML) + if err != nil { + return nil, err + } + + if _, err := toml.NewDecoder(bytes.NewBuffer(registriesTOML)).Decode(®istriesConf); err != nil { + return nil, fmt.Errorf("error decoding registries config: %w", err) + } } - for scope := range clusterScopePolicies { - registriesDockerConfig[scope] = sigstoreAttachment + + for policyScope := range clusterScopePolicies { + registriesDockerConfig[policyScope] = sigstoreAttachment + if registriesTOML == nil { + continue + } + sysContext := &types.SystemContext{ + SystemRegistriesConfPath: tmpFile.Name(), + SystemRegistriesConfDirPath: os.DevNull, + } + parentReg, err := sysregistriesv2.FindRegistry(sysContext, policyScope) + if err != nil { + return nil, err + } + + if err = addScopeMirrorsSigstoreRegistriesdConfig(registriesDockerConfig, policyScope, parentReg, sigstoreAttachment); err != nil { + return nil, fmt.Errorf("error adding clusterimagepolicy scope %s relevant mirrors: %w", policyScope, err) + } + for _, reg := range registriesConf.Registries { + scope := reg.Location + if reg.Prefix != "" { + scope = reg.Prefix + } + if runtimeutils.ScopeIsNestedInsideScope(scope, policyScope) && scope != policyScope { + nestedReg, err := sysregistriesv2.FindRegistry(sysContext, scope) + if err != nil { + return nil, err + } + if err = addScopeMirrorsSigstoreRegistriesdConfig(registriesDockerConfig, scope, nestedReg, sigstoreAttachment); err != nil { + return nil, fmt.Errorf("error adding clusterimagepolicy scope %s relevant mirrors: %w", policyScope, err) + } + } + } } registriesConfig := ®istriesConfig{} @@ -954,3 +1011,68 @@ func generateSigstoreRegistriesdConfig(clusterScopePolicies map[string]signature } return data, nil } + +func addScopeMirrorsSigstoreRegistriesdConfig(registriesDockerConfig map[string]dockerConfig, scope string, reg *sysregistriesv2.Registry, sigstoreAttachment dockerConfig) error { + if reg == nil { + return nil + } + dummyDigest := "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + dummyTag := "aaa" + dummyPath := "/a" + dummyPrefix := "matched" + addedDummy := false + + if strings.HasPrefix(scope, "*.") { + scope = strings.Replace(scope, "*", dummyPrefix, 1) + } + + if !strings.Contains(scope, "/") { + scope += dummyPath + addedDummy = true + } + + scopeRef, err := reference.Parse(scope) + if err != nil { + return fmt.Errorf("error parsing scope %s: %w", scope, err) + } + + namedScopeRef, ok := scopeRef.(reference.Named) + if !ok { + return fmt.Errorf("scope %s is not a named reference", scope) + } + repo := reference.TrimNamed(namedScopeRef) + + d, _ := digest.Parse(dummyDigest) + digestRef, err := reference.WithDigest(repo, d) + if err != nil { + return fmt.Errorf("error parsing digest name for scope %s: %w", scope, err) + } + tagRef, err := reference.WithTag(repo, dummyTag) + if err != nil { + return fmt.Errorf("error parsing tag name for scope %s: %w", scope, err) + } + + digestSources, err := reg.PullSourcesFromReference(digestRef) + if err != nil { + return fmt.Errorf("error getting digest sources for scope %s: %w", scope, err) + } + tagSources, err := reg.PullSourcesFromReference(tagRef) + if err != nil { + return fmt.Errorf("error getting tag sources for scope %s: %w", scope, err) + } + for _, s := range append(digestSources, tagSources...) { + endpoint := s.Reference.Name() + if addedDummy { + endpoint = strings.TrimSuffix(endpoint, dummyPath) + } + // "Location" is empty if the matched Registry source containers a wildcard, only add mirrors in this case + // caller has unconditionally configured attachments for ClusterImagePolicy scopes in registriesDockerConfig[policyScope]. + // If a dummyPrefix was added to process a wildcard Registry entry, we only want to configure the mirror endpoints, if any; + // we do not want to add a new entry for exactly dummyPrefix + originalScope. + if s.Endpoint.Location == "" { + continue + } + registriesDockerConfig[endpoint] = sigstoreAttachment + } + return nil +} diff --git a/pkg/controller/container-runtime-config/helpers_test.go b/pkg/controller/container-runtime-config/helpers_test.go index 6754409c50..157d59f897 100644 --- a/pkg/controller/container-runtime-config/helpers_test.go +++ b/pkg/controller/container-runtime-config/helpers_test.go @@ -20,9 +20,11 @@ import ( apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/yaml" ) func TestUpdateRegistriesConfig(t *testing.T) { @@ -523,6 +525,23 @@ func clusterImagePolicyTestCRs() map[string]apicfgv1alpha1.ClusterImagePolicy { }, }, }, + "test-cr2": { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cr2", + }, + Spec: apicfgv1alpha1.ClusterImagePolicySpec{ + Scopes: []apicfgv1alpha1.ImageScope{"a.com/a1/a2", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "*.example.com", "policy.scope", "foo.example.com/ns/repo"}, + Policy: apicfgv1alpha1.Policy{ + RootOfTrust: apicfgv1alpha1.PolicyRootOfTrust{ + PolicyType: apicfgv1alpha1.PublicKeyRootOfTrust, + PublicKey: &apicfgv1alpha1.PublicKey{ + KeyData: testKeyData, + RekorKeyData: testRekorKeyData, + }, + }, + }, + }, + }, } return testClusterImagePolicyCRs } @@ -1245,22 +1264,144 @@ func TestGetValidScopePolicies(t *testing.T) { } func TestGenerateSigstoreRegistriesConfig(t *testing.T) { - testClusterImagePolicyCR0 := clusterImagePolicyTestCRs()["test-cr0"] - testClusterImagePolicyCR1 := clusterImagePolicyTestCRs()["test-cr1"] - expectSigstoreRegistriesConfig := []byte( - `docker: - test0.com: - use-sigstore-attachments: true - test1.com: - use-sigstore-attachments: true -`) + // testcase CIP scopes: + // "a.com/a1/a2", + // "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", + // "*.example.com", + // "foo.example.com/ns/repo", + // "policy.scope" + testClusterImagePolicyCR2 := clusterImagePolicyTestCRs()["test-cr2"] - clusterScopePolicies, err := getValidScopePolicies([]*apicfgv1alpha1.ClusterImagePolicy{&testClusterImagePolicyCR0, &testClusterImagePolicyCR1}) - require.NoError(t, err) - got, err := generateSigstoreRegistriesdConfig(clusterScopePolicies) + clusterScopePolicies, err := getValidScopePolicies([]*apicfgv1alpha1.ClusterImagePolicy{&testClusterImagePolicyCR2}) require.NoError(t, err) - require.Equal(t, string(expectSigstoreRegistriesConfig), string(got)) + + type testcase struct { + name string + clusterScopePolicies map[string]signature.PolicyRequirements + icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy + idmsRules []*apicfgv1.ImageDigestMirrorSet + itmsRules []*apicfgv1.ImageTagMirrorSet + expectedSigstoreRegistriesConfigScopes []string + } + + testcases := []testcase{ + { + name: "cip objects exist, icsp/idms/itms are empty", + clusterScopePolicies: clusterScopePolicies, + expectedSigstoreRegistriesConfigScopes: []string{"*.example.com", "a.com/a1/a2", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "foo.example.com/ns/repo", "policy.scope"}, + }, + { + name: "cip has no revelant registries mirrors", + clusterScopePolicies: clusterScopePolicies, + icspRules: []*apioperatorsv1alpha1.ImageContentSourcePolicy{ + { + Spec: apioperatorsv1alpha1.ImageContentSourcePolicySpec{ + RepositoryDigestMirrors: []apioperatorsv1alpha1.RepositoryDigestMirrors{ + {Source: "x.com/x1/x2", Mirrors: []string{"x-x1-x2.mirror/x1/x2"}}, + }, + }, + }, + }, + idmsRules: []*apicfgv1.ImageDigestMirrorSet{ + { + Spec: apicfgv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []apicfgv1.ImageDigestMirrors{ // other.com is neither insecure nor blocked + {Source: "y.com/y1/y2", Mirrors: []apicfgv1.ImageMirror{"y-y1-y2.mirror/y1/y2"}}, + }, + }, + }, + }, + itmsRules: []*apicfgv1.ImageTagMirrorSet{ + { + Spec: apicfgv1.ImageTagMirrorSetSpec{ + ImageTagMirrors: []apicfgv1.ImageTagMirrors{ + {Source: "z.com/z1/z2", Mirrors: []apicfgv1.ImageMirror{"z-z1-z2.mirror/z1/z2"}}, + }, + }, + }, + }, + expectedSigstoreRegistriesConfigScopes: []string{"*.example.com", "a.com/a1/a2", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "foo.example.com/ns/repo", "policy.scope"}, + }, + { + name: "cip scope a.com/a1/a2 is super scope of registries sources", + clusterScopePolicies: clusterScopePolicies, + idmsRules: []*apicfgv1.ImageDigestMirrorSet{ + { + Spec: apicfgv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []apicfgv1.ImageDigestMirrors{ + {Source: "a.com/a1/a2", Mirrors: []apicfgv1.ImageMirror{"a-a1-a2.mirror/a1/a2"}}, + {Source: "a.com/a1/a2/a3", Mirrors: []apicfgv1.ImageMirror{"a-a1-a2-a3.mirror/m1/m2/m3"}}, + {Source: "a.com/a1/a2/a3-1", Mirrors: []apicfgv1.ImageMirror{"a-a1-a2-a3-1.mirror/m1/m2/m3"}}, + {Source: "x.com/x1/x2", Mirrors: []apicfgv1.ImageMirror{"x-x1-x2.mirror/m1/m2"}}, + }, + }, + }, + }, + expectedSigstoreRegistriesConfigScopes: []string{"*.example.com", "a-a1-a2-a3-1.mirror/m1/m2/m3", "a-a1-a2-a3.mirror/m1/m2/m3", "a-a1-a2.mirror/a1/a2", "a.com/a1/a2", "a.com/a1/a2/a3", "a.com/a1/a2/a3-1", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "foo.example.com/ns/repo", "policy.scope"}, + }, + { + name: "cip scope a.com/a1/a2 nested in registries sources", + clusterScopePolicies: clusterScopePolicies, + idmsRules: []*apicfgv1.ImageDigestMirrorSet{ + { + Spec: apicfgv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []apicfgv1.ImageDigestMirrors{ + {Source: "a.com", Mirrors: []apicfgv1.ImageMirror{"a.mirror"}}, + {Source: "a.com/a1", Mirrors: []apicfgv1.ImageMirror{"a-a1.mirror/a1"}}, + {Source: "x.com/x1/x2", Mirrors: []apicfgv1.ImageMirror{"x-x1-x2.mirror/x1/x2"}}, + }, + }, + }, + }, + expectedSigstoreRegistriesConfigScopes: []string{"*.example.com", "a-a1.mirror/a1/a2", "a.com/a1/a2", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "policy.scope", "foo.example.com/ns/repo"}, + }, + { + name: "cip wildcard scope *.example.com is the mirror source", + clusterScopePolicies: clusterScopePolicies, + idmsRules: []*apicfgv1.ImageDigestMirrorSet{ + { + Spec: apicfgv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []apicfgv1.ImageDigestMirrors{ + {Source: "*.example.com", Mirrors: []apicfgv1.ImageMirror{"our.mirror/example"}}, + }, + }, + }, + }, + expectedSigstoreRegistriesConfigScopes: []string{"*.example.com", "a.com/a1/a2", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "policy.scope", "foo.example.com/ns/repo", "our.mirror/example/ns/repo", "our.mirror/example"}, + }, + { + name: "cip scope *.example.com and mirror source have wildcard matching", + clusterScopePolicies: clusterScopePolicies, + idmsRules: []*apicfgv1.ImageDigestMirrorSet{ + { + Spec: apicfgv1.ImageDigestMirrorSetSpec{ + ImageDigestMirrors: []apicfgv1.ImageDigestMirrors{ + {Source: "a.example.com", Mirrors: []apicfgv1.ImageMirror{"a-example.mirror"}}, + {Source: "*.x.example.com", Mirrors: []apicfgv1.ImageMirror{"matched.example.mirror", "star-x.example.mirror"}}, + {Source: "*.scope", Mirrors: []apicfgv1.ImageMirror{"start-scope.mirror"}}, + {Source: "x/x1/x2", Mirrors: []apicfgv1.ImageMirror{"x-x1-x2.mirror/x1/x2"}}, + }, + }, + }, + }, + expectedSigstoreRegistriesConfigScopes: []string{"*.example.com", "a-example.mirror", "a.com/a1/a2", "a.com/a1/a2@sha256:0000000000000000000000000000000000000000000000000000000000000000", "a.example.com", "star-x.example.mirror", "matched.example.mirror", "policy.scope", "start-scope.mirror", "foo.example.com/ns/repo"}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + registriesTOML, err := updateRegistriesConfig(templateRegistriesConfig, nil, nil, tc.icspRules, tc.idmsRules, tc.itmsRules) + require.NoError(t, err) + got, err := generateSigstoreRegistriesdConfig(tc.clusterScopePolicies, registriesTOML) + require.NoError(t, err) + + gotRegistriesCfg := ®istriesConfig{} + err = yaml.Unmarshal(got, &gotRegistriesCfg) + require.NoError(t, err) + assert.ElementsMatch(t, tc.expectedSigstoreRegistriesConfigScopes, maps.Keys(gotRegistriesCfg.Docker)) + }) + } } func TestGeneratePolicyJSON(t *testing.T) {