Skip to content

Commit

Permalink
feat(auth): add oidc support by oauth2-proxy and refactor with operat…
Browse files Browse the repository at this point in the history
…or-go
  • Loading branch information
whg517 committed Sep 25, 2024
1 parent 48e6afc commit f92b89c
Show file tree
Hide file tree
Showing 23 changed files with 679 additions and 16 deletions.
2 changes: 1 addition & 1 deletion .chainsaw.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ kind: Configuration
metadata:
name: custom-config
spec:
#namespace: test
timeouts:
apply: 120s
assert: 300s
Expand All @@ -13,3 +12,4 @@ spec:
exec: 200s
skipDelete: false
failFast: true
parallel: 1
4 changes: 3 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import (
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

authv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/authentication/v1alpha1"
listenerv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/listeners/v1alpha1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
Expand All @@ -38,7 +40,6 @@ import (

hdfsv1alpha1 "github.com/zncdatadev/hdfs-operator/api/v1alpha1"
"github.com/zncdatadev/hdfs-operator/internal/controller"
listenerv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/listeners/v1alpha1"
//+kubebuilder:scaffold:imports
)

Expand All @@ -50,6 +51,7 @@ var (
func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))

utilruntime.Must(authv1alpha1.AddToScheme(scheme))
utilruntime.Must(hdfsv1alpha1.AddToScheme(scheme))
utilruntime.Must(listenerv1alpha1.AddToScheme(scheme))
//+kubebuilder:scaffold:scheme
Expand Down
9 changes: 9 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ rules:
- ""
resources:
- configmaps
- secrets
- serviceaccounts
- services
verbs:
Expand Down Expand Up @@ -38,6 +39,14 @@ rules:
- patch
- update
- watch
- apiGroups:
- authentication.zncdata.dev
resources:
- authenticationclasses
verbs:
- get
- list
- watch
- apiGroups:
- hdfs.zncdata.dev
resources:
Expand Down
229 changes: 229 additions & 0 deletions internal/common/oidc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
package common

import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"net/url"
"strconv"
"strings"

hdfsv1alpha1 "github.com/zncdatadev/hdfs-operator/api/v1alpha1"
authv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/authentication/v1alpha1"
"github.com/zncdatadev/operator-go/pkg/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
)

var (
oidcLogger = ctrl.Log.WithName("oidc")
)

func MakeOidcContainer(
ctx context.Context,
client ctrlclient.Client,
instance *hdfsv1alpha1.HdfsCluster,
port int32,
) (*corev1.Container, error) {
authClass := &authv1alpha1.AuthenticationClass{}
if err := client.Get(ctx, ctrlclient.ObjectKey{Namespace: instance.Namespace, Name: instance.Spec.ClusterConfigSpec.Authentication.AuthenticationClass}, authClass); err != nil {
if ctrlclient.IgnoreNotFound(err) != nil {
return nil, err
}
return nil, nil
}

if authClass.Spec.AuthenticationProvider.OIDC == nil || instance.Spec.ClusterConfigSpec.Authentication.Oidc == nil {
oidcLogger.Info("OIDC provider is not configured", "OidcProvider", authClass.Spec.AuthenticationProvider.OIDC, "OidcCredential", instance.Spec.ClusterConfigSpec.Authentication.Oidc)
return nil, nil
}

oidc := NewOidcContainerBuilder(
client,
instance,
authClass.Spec.AuthenticationProvider.OIDC,
instance.Spec.ClusterConfigSpec.Authentication.Oidc,
port,
)
obj := oidc.Build(oidc)
return &obj, nil
}

type OidcContainerBuilder struct {
ContainerBuilder
client client.Client
instanceUid string
port int32
oidcProvider *authv1alpha1.OIDCProvider
oidc *hdfsv1alpha1.OidcSpec
}

func NewOidcContainerBuilder(
client client.Client,
instance *hdfsv1alpha1.HdfsCluster,
oidcProvider *authv1alpha1.OIDCProvider,
oidc *hdfsv1alpha1.OidcSpec,
port int32,
) *OidcContainerBuilder {
imageSpec := instance.Spec.Image
image := hdfsv1alpha1.TransformImage(imageSpec)
return &OidcContainerBuilder{
ContainerBuilder: *NewContainerBuilder(image.String(), image.GetPullPolicy(), corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("512Mi"),
},
}),
client: client,
instanceUid: string(instance.UID),
port: port,
oidcProvider: oidcProvider,
oidc: oidc,
}
}

func (o *OidcContainerBuilder) ContainerName() string {
return "oidc"
}

func (o *OidcContainerBuilder) ContainerPorts() []corev1.ContainerPort {
return []corev1.ContainerPort{
{
Name: "oidc",
ContainerPort: 4180,
},
}
}

func (o *OidcContainerBuilder) ContainerEnv() []corev1.EnvVar {

oidcProvider := o.oidcProvider

scopes := []string{"openid", "email", "profile"}

if o.oidc.ExtraScopes != nil {
scopes = append(scopes, o.oidc.ExtraScopes...)
}

issuer := url.URL{
Scheme: "http",
Host: oidcProvider.Hostname,
Path: oidcProvider.RootPath,
}

if oidcProvider.Port != 0 && oidcProvider.Port != 80 {
issuer.Host += ":" + strconv.Itoa(oidcProvider.Port)
}

provisioner := oidcProvider.Provisioner
// TODO: fix support keycloak-oidc
if provisioner == "keycloak" {
provisioner = "keycloak-oidc"
}

clientCredentialsSecretName := o.oidc.ClientCredentialsSecret

hash := sha256.Sum256([]byte(o.instanceUid))
hashStr := hex.EncodeToString(hash[:])
tokenBytes := []byte(hashStr[:16])

cookieSecret := base64.StdEncoding.EncodeToString([]byte(base64.StdEncoding.EncodeToString(tokenBytes)))

return []corev1.EnvVar{
{
Name: "OAUTH2_PROXY_COOKIE_SECRET",
Value: cookieSecret,
},
{
Name: "OAUTH2_PROXY_CLIENT_ID",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: clientCredentialsSecretName,
},
Key: "CLIENT_ID",
},
},
},
{
Name: "OAUTH2_PROXY_CLIENT_SECRET",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: clientCredentialsSecretName,
},
Key: "CLIENT_SECRET",
},
},
},
{
Name: "POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
{
Name: "OAUTH2_PROXY_OIDC_ISSUER_URL",
Value: issuer.String(),
},
{
Name: "OAUTH2_PROXY_SCOPE",
Value: strings.Join(scopes, " "),
},
{
Name: "OAUTH2_PROXY_PROVIDER",
Value: provisioner,
},
{
Name: "UPSTREAM",
Value: "http://$(POD_IP):" + strconv.Itoa(int(o.port)),
},
{
Name: "OAUTH2_PROXY_HTTP_ADDRESS",
Value: "0.0.0.0:4180",
},
{
Name: "OAUTH2_PROXY_REDIRECT_URL",
Value: "http://localhost:4180/oauth2/callback",
},
{
Name: "OAUTH2_PROXY_CODE_CHALLENGE_METHOD",
Value: "S256",
},
{
Name: "OAUTH2_PROXY_EMAIL_DOMAINS",
Value: "*",
},
}

}

func (o *OidcContainerBuilder) Command() []string {
script := `
set -ex
ARCH=$(uname -m)
ARCH="${ARCH/x86_64/amd64}"
ARCH="${ARCH/aarch64/arm64}"
mkdir /kubedoop/oauth2-proxy
cd /kubedoop/oauth2-proxy
curl -sSfL \
https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v7.6.0/oauth2-proxy-v7.6.0.linux-${ARCH}.tar.gz \
| tar xzf - --strip-components=1
/kubedoop/oauth2-proxy/oauth2-proxy \
--upstream=${UPSTREAM}
`
return []string{
"bash",
"-c",
util.IndentTab4Spaces(script),
}
}
16 changes: 14 additions & 2 deletions internal/controller/data/statefulset.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func NewStatefulSet(
}
}

func (s *StatefulSetReconciler) Build(_ context.Context) (client.Object, error) {
func (s *StatefulSetReconciler) Build(ctx context.Context) (client.Object, error) {
sts := &appv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: createStatefulSetName(s.Instance.GetName(), s.GroupName),
Expand Down Expand Up @@ -80,10 +80,22 @@ func (s *StatefulSetReconciler) Build(_ context.Context) (client.Object, error)
img := hdfsv1alpha1.TransformImage(s.Instance.Spec.Image)
common.ExtendStatefulSetByVector(nil, sts, img, createConfigName(s.Instance.GetName(), s.GroupName))
}

if s.Instance.Spec.ClusterConfigSpec.Authentication != nil && s.Instance.Spec.ClusterConfigSpec.Authentication.AuthenticationClass != "" {
oidcContainer, err := common.MakeOidcContainer(ctx, s.Client, s.Instance, s.getHttpPort())
if err != nil {
return nil, err
}
if oidcContainer != nil {
sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, *oidcContainer)
}
}
return sts, nil
}

func (s *StatefulSetReconciler) getHttpPort() int32 {
return common.HttpPort(s.Instance.Spec.ClusterConfigSpec, hdfsv1alpha1.DataNodeHttpsPort, hdfsv1alpha1.DataNodeHttpPort).ContainerPort
}

func (s *StatefulSetReconciler) SetAffinity(resource client.Object) {
dep := resource.(*appv1.StatefulSet)
if affinity := s.MergedCfg.Config.Affinity; affinity != nil {
Expand Down
6 changes: 6 additions & 0 deletions internal/controller/data/svc_headless.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ func (s *ServiceReconciler) makePorts() []corev1.ServicePort {
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(hdfsv1alpha1.IpcName),
},
{
Name: "oidc",
Port: 4180,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString("oidc"),
},
}
return append(ports, common.ServiceHttpPort(s.Instance.Spec.ClusterConfigSpec, ServiceHttpsPort, ServiceHttpPort))
}
2 changes: 2 additions & 0 deletions internal/controller/hdfscluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@ type HdfsClusterReconciler struct {
//+kubebuilder:rbac:groups=hdfs.zncdata.dev,resources=hdfsclusters/finalizers,verbs=update
//+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
//+kubebuilder:rbac:groups=authentication.zncdata.dev,resources=authenticationclasses,verbs=get;list;watch

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
Expand Down
16 changes: 14 additions & 2 deletions internal/controller/journal/statefulset.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func NewStatefulSet(
}
}

func (s *StatefulSetReconciler) Build(_ context.Context) (client.Object, error) {
func (s *StatefulSetReconciler) Build(ctx context.Context) (client.Object, error) {
sts := &appv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: createStatefulSetName(s.Instance.GetName(), s.GroupName),
Expand Down Expand Up @@ -74,10 +74,22 @@ func (s *StatefulSetReconciler) Build(_ context.Context) (client.Object, error)
img := hdfsv1alpha1.TransformImage(s.Instance.Spec.Image)
common.ExtendStatefulSetByVector(nil, sts, img, createConfigName(s.Instance.GetName(), s.GroupName))
}

if s.Instance.Spec.ClusterConfigSpec.Authentication != nil && s.Instance.Spec.ClusterConfigSpec.Authentication.AuthenticationClass != "" {
oidcContainer, err := common.MakeOidcContainer(ctx, s.Client, s.Instance, s.getHttpPort())
if err != nil {
return nil, err
}
if oidcContainer != nil {
sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, *oidcContainer)
}
}
return sts, nil
}

func (s *StatefulSetReconciler) getHttpPort() int32 {
return common.HttpPort(s.Instance.Spec.ClusterConfigSpec, hdfsv1alpha1.JournalNodeHttpsPort, hdfsv1alpha1.JournalNodeHttpPort).ContainerPort
}

func (s *StatefulSetReconciler) SetAffinity(resource client.Object) {
dep := resource.(*appv1.StatefulSet)
if affinity := s.MergedCfg.Config.Affinity; affinity != nil {
Expand Down
6 changes: 6 additions & 0 deletions internal/controller/journal/svc_headless.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ func (s *ServiceReconciler) makePorts() []corev1.ServicePort {
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(hdfsv1alpha1.RpcName),
},
{
Name: "oidc",
Port: 4180,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString("oidc"),
},
}
return append(ports, common.ServiceHttpPort(s.Instance.Spec.ClusterConfigSpec, ServiceHttpsPort, ServiceHttpPort))
}
Loading

0 comments on commit f92b89c

Please sign in to comment.