Skip to content

Commit

Permalink
add support for credential provider plugin (#1383)
Browse files Browse the repository at this point in the history
Signed-off-by: Kuromesi <blackfacepan@163.com>
  • Loading branch information
Kuromesi committed Jun 11, 2024
1 parent 145a9af commit 5ea03f1
Show file tree
Hide file tree
Showing 9 changed files with 277 additions and 62 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ lint: golangci-lint ## Run golangci-lint against code.

test: generate fmt vet manifests envtest ## Run tests
echo $(ENVTEST)
go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./pkg/... -coverprofile cover.out
rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin

coverage-report: ## Generate cover.html from cover.out
go tool cover -html=cover.out -o cover.html
Expand Down
27 changes: 24 additions & 3 deletions cmd/daemon/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ limitations under the License.
package main

import (
"os"

"k8s.io/kubernetes/pkg/credentialprovider/plugin"

"flag"
"math/rand"
"net/http"
Expand All @@ -34,12 +38,15 @@ import (
"github.com/openkruise/kruise/pkg/daemon"
"github.com/openkruise/kruise/pkg/features"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"github.com/openkruise/kruise/pkg/util/secret"
)

var (
bindAddr = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.")
pprofAddr = flag.String("pprof-addr", ":10222", "The address the pprof binds to.")
enablePprof = flag.Bool("enable-pprof", true, "Enable pprof for daemon.")
bindAddr = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.")
pprofAddr = flag.String("pprof-addr", ":10222", "The address the pprof binds to.")
enablePprof = flag.Bool("enable-pprof", true, "Enable pprof for daemon.")
pluginConfigFile = flag.String("plugin-config-file", "/kruise/CredentialProviderPlugin.yaml", "The path of plugin config file.")
pluginBinDir = flag.String("plugin-bin-dir", "/kruise/plugins", "The path of directory of plugin binaries.")
)

func main() {
Expand Down Expand Up @@ -68,6 +75,20 @@ func main() {
if err != nil {
klog.Fatalf("Failed to new daemon: %v", err)
}

if _, err := os.Stat(*pluginConfigFile); err == nil {
err = plugin.RegisterCredentialProviderPlugins(*pluginConfigFile, *pluginBinDir)
if err != nil {
klog.Errorf("Failed to register credential provider plugins: %v", err)
}
} else if os.IsNotExist(err) {
klog.Infof("No plugin config file found, skipping: %s", *pluginConfigFile)
} else {
klog.Errorf("Failed to check plugin config file: %v", err)
}
// make sure the new docker key ring is made and set after the credential plugins are registered
secret.MakeAndSetKeyring()

if err := d.Run(ctx); err != nil {
klog.Fatalf("Failed to start daemon: %v", err)
}
Expand Down
44 changes: 22 additions & 22 deletions pkg/daemon/criruntime/imageruntime/cri.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,31 +131,31 @@ func (c *commonCRIImageService) pullImageV1(ctx context.Context, imageName, tag
// for some runtime implementations.
pullImageReq.SandboxConfig.Annotations[pullingImageSandboxConfigAnno] = "kruise-daemon"

if len(pullSecrets) > 0 {
var authInfos []daemonutil.AuthInfo
authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, repoToPull)
if err == nil {
var pullErrs []error
for _, authInfo := range authInfos {
var pullErr error
klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
pullImageReq.Auth = &runtimeapi.AuthConfig{
Username: authInfo.Username,
Password: authInfo.Password,
}
_, pullErr = c.criImageClient.PullImage(ctx, pullImageReq)
if pullErr == nil {
pipeW.CloseWithError(io.EOF)
return newImagePullStatusReader(pipeR), nil
}
klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
pullErrs = append(pullErrs, pullErr)

var authInfos []daemonutil.AuthInfo
authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, repoToPull)
if err == nil {
var pullErrs []error
for _, authInfo := range authInfos {
var pullErr error
klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
pullImageReq.Auth = &runtimeapi.AuthConfig{
Username: authInfo.Username,
Password: authInfo.Password,
}
if len(pullErrs) > 0 {
err = utilerrors.NewAggregate(pullErrs)
_, pullErr = c.criImageClient.PullImage(ctx, pullImageReq)
if pullErr == nil {
pipeW.CloseWithError(io.EOF)
return newImagePullStatusReader(pipeR), nil
}
klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
pullErrs = append(pullErrs, pullErr)

}
if len(pullErrs) > 0 {
err = utilerrors.NewAggregate(pullErrs)
}
} else {
klog.Errorf("Failed to convert to auth info for registry, err %v", err)
}

// Try the default secret
Expand Down
36 changes: 18 additions & 18 deletions pkg/daemon/criruntime/imageruntime/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,26 +81,26 @@ func (d *dockerImageService) PullImage(ctx context.Context, imageName, tag strin
fullName := imageName + ":" + tag
var ioReader io.ReadCloser

if len(pullSecrets) > 0 {
var authInfos []daemonutil.AuthInfo
authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry)
if err == nil {
var pullErrs []error
for _, authInfo := range authInfos {
var pullErr error
klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
ioReader, pullErr = d.client.ImagePull(ctx, fullName, dockertypes.ImagePullOptions{RegistryAuth: authInfo.EncodeToString()})
if pullErr == nil {
return newImagePullStatusReader(ioReader), nil
}
d.handleRuntimeError(pullErr)
klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
pullErrs = append(pullErrs, pullErr)
}
if len(pullErrs) > 0 {
err = utilerrors.NewAggregate(pullErrs)
var authInfos []daemonutil.AuthInfo
authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry)
if err == nil {
var pullErrs []error
for _, authInfo := range authInfos {
var pullErr error
klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username)
ioReader, pullErr = d.client.ImagePull(ctx, fullName, dockertypes.ImagePullOptions{RegistryAuth: authInfo.EncodeToString()})
if pullErr == nil {
return newImagePullStatusReader(ioReader), nil
}
d.handleRuntimeError(pullErr)
klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr)
pullErrs = append(pullErrs, pullErr)
}
if len(pullErrs) > 0 {
err = utilerrors.NewAggregate(pullErrs)
}
} else {
klog.Errorf("Failed to convert to auth info for registry, err %v", err)
}

// Try the default secret
Expand Down
152 changes: 152 additions & 0 deletions pkg/daemon/criruntime/imageruntime/fake_plugin/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package main

import (
"bufio"
"context"
"errors"
"fmt"
"io"
"os"
"strings"

"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/component-base/logs"
"k8s.io/klog/v2"
"k8s.io/kubelet/pkg/apis/credentialprovider/install"
v1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
)

var (
scheme = runtime.NewScheme()
codecs = serializer.NewCodecFactory(scheme)
)

func init() {
install.Install(scheme)
}

func getCredentials(ctx context.Context, image string, args []string) (*v1.CredentialProviderResponse, error) {
response := &v1.CredentialProviderResponse{
CacheKeyType: v1.RegistryPluginCacheKeyType,
Auth: map[string]v1.AuthConfig{
"registry.plugin.com/test": {
Username: "user",
Password: "password",
},
},
}
return response, nil
}

func runPlugin(ctx context.Context, r io.Reader, w io.Writer, args []string) error {
data, err := io.ReadAll(r)
if err != nil {
return err
}

_, err = json.DefaultMetaFactory.Interpret(data)
if err != nil {
return err
}

request, err := decodeRequest(data)
if err != nil {
return err
}

if request.Image == "" {
return errors.New("image in plugin request was empty")
}

// Deny all requests except for those where the image URL contains registry.plugin.com
// to test whether kruise could get expected auths if plugin fails to run
if !strings.Contains(request.Image, "registry.plugin.com") {
return errors.New("image in plugin request not supported: " + request.Image)
}

response, err := getCredentials(ctx, request.Image, args)
if err != nil {
return err
}

if response == nil {
return errors.New("CredentialProviderResponse from plugin was nil")
}

encodedResponse, err := encodeResponse(response)
if err != nil {
return err
}

writer := bufio.NewWriter(w)
defer writer.Flush()
if _, err := writer.Write(encodedResponse); err != nil {
return err
}

return nil
}

func decodeRequest(data []byte) (*v1.CredentialProviderRequest, error) {
obj, gvk, err := codecs.UniversalDecoder(v1.SchemeGroupVersion).Decode(data, nil, nil)
if err != nil {
return nil, err
}

if gvk.Kind != "CredentialProviderRequest" {
return nil, fmt.Errorf("kind was %q, expected CredentialProviderRequest", gvk.Kind)
}

if gvk.Group != v1.GroupName {
return nil, fmt.Errorf("group was %q, expected %s", gvk.Group, v1.GroupName)
}

request, ok := obj.(*v1.CredentialProviderRequest)
if !ok {
return nil, fmt.Errorf("unable to convert %T to *CredentialProviderRequest", obj)
}

return request, nil
}

func encodeResponse(response *v1.CredentialProviderResponse) ([]byte, error) {
mediaType := "application/json"
info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType)
if !ok {
return nil, fmt.Errorf("unsupported media type %q", mediaType)
}

encoder := codecs.EncoderForVersion(info.Serializer, v1.SchemeGroupVersion)
data, err := runtime.Encode(encoder, response)
if err != nil {
return nil, fmt.Errorf("failed to encode response: %v", err)
}

return data, nil
}

func main() {
logs.InitLogs()
defer logs.FlushLogs()

if err := newCredentialProviderCommand().Execute(); err != nil {
os.Exit(1)
}
}

func newCredentialProviderCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "acr-credential-provider",
Short: "ACR credential provider for kubelet",
Run: func(cmd *cobra.Command, args []string) {
if err := runPlugin(context.TODO(), os.Stdin, os.Stdout, os.Args[1:]); err != nil {
klog.Errorf("Error running credential provider plugin: %v", err)
os.Exit(1)
}
},
}
return cmd
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: kubelet.config.k8s.io/v1
kind: CredentialProviderConfig
providers:
- name: fake-credential-plugin
matchImages:
- "registry.plugin.com"
- "registry.private.com"
defaultCacheDuration: "12h"
apiVersion: credentialprovider.kubelet.k8s.io/v1
21 changes: 21 additions & 0 deletions pkg/daemon/criruntime/imageruntime/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"testing"

v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/credentialprovider/plugin"
"k8s.io/kubernetes/pkg/util/parsers"

"github.com/openkruise/kruise/pkg/util/secret"
Expand Down Expand Up @@ -103,7 +105,26 @@ func TestMatchRegistryAuths(t *testing.T) {
},
ExpectMinValue: 0,
},
{
name: "test credential plugin if matched",
Image: "registry.plugin.com/test/echoserver:v1",
GetSecrets: func() []v1.Secret {
return []v1.Secret{}
},
ExpectMinValue: 1,
},
}
pluginBinDir := "fake_plugin"
pluginConfigFile := "fake_plugin/plugin-config.yaml"
// credential plugin is configured for images with "registry.plugin.com" and "registry.private.com",
// however, only images with "registry.plugin.com" will return a fake credential,
// other images will be denied by the plugin and an error will be raised,
// this is to test whether kruise could get expected auths if plugin fails to run
err := plugin.RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir)
if err != nil {
klog.Errorf("Failed to register credential provider plugins: %v", err)
}
secret.MakeAndSetKeyring()
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
repoToPull, _, _, err := parsers.ParseImageName(cs.Image)
Expand Down
Loading

0 comments on commit 5ea03f1

Please sign in to comment.