Skip to content

Commit

Permalink
add long term token to datavolume
Browse files Browse the repository at this point in the history 
Signed-off-by: Michael Henriksen <mhenriks@redhat.com>
  • Loading branch information
@mhenriks
mhenriks committed Jun 27, 2023
1 parent 691028e commit 75819aa
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 48 deletions.
54 changes: 48 additions & 6 deletions pkg/controller/datavolume/clone-controller-base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,8 @@ type CloneReconcilerBase struct {
pullPolicy string
cloneSourceAPIGroup *string
cloneSourceKind string
tokenValidator token.Validator
shortTokenValidator token.Validator
longTokenValidator token.Validator
tokenGenerator token.Generator
}

Expand Down Expand Up @@ -203,25 +204,66 @@ func (r *CloneReconcilerBase) updatePVCForPopulation(dataVolume *cdiv1.DataVolum
return nil
}

func (r *CloneReconcilerBase) ensureExtendedToken(pvc *corev1.PersistentVolumeClaim) error {
func (r *CloneReconcilerBase) ensureExtendedTokenDV(dv *cdiv1.DataVolume) (bool, error) {
if !isCrossNamespaceClone(dv) {
return false, nil
}

_, ok := dv.Annotations[cc.AnnExtendedCloneToken]
if ok {
return false, nil
}

token, ok := dv.Annotations[cc.AnnCloneToken]
if !ok {
return false, fmt.Errorf("token missing")
}

payload, err := r.shortTokenValidator.Validate(token)
if err != nil {
return false, err
}

if payload.Params == nil {
payload.Params = make(map[string]string)
}
payload.Params["uid"] = string(dv.UID)

newToken, err := r.tokenGenerator.Generate(payload)
if err != nil {
return false, err
}

dv.Annotations[cc.AnnExtendedCloneToken] = newToken

return true, nil
}

func (r *CloneReconcilerBase) ensureExtendedTokenPVC(dv *cdiv1.DataVolume, pvc *corev1.PersistentVolumeClaim) error {
if !isCrossNamespaceClone(dv) {
return nil
}

_, ok := pvc.Annotations[cc.AnnExtendedCloneToken]
if ok {
return nil
}

token, ok := pvc.Annotations[cc.AnnCloneToken]
token, ok := dv.Annotations[cc.AnnExtendedCloneToken]
if !ok {
return fmt.Errorf("token missing")
}

payload, err := r.tokenValidator.Validate(token)
payload, err := r.longTokenValidator.Validate(token)
if err != nil {
return err
}

if payload.Params == nil {
payload.Params = make(map[string]string)
if payload.Params["uid"] != string(dv.UID) {
return fmt.Errorf("token uid mismatch")
}

// now use pvc uid
payload.Params["uid"] = string(pvc.UID)

newToken, err := r.tokenGenerator.Generate(payload)
Expand Down
35 changes: 25 additions & 10 deletions pkg/controller/datavolume/pvc-clone-controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,12 @@ func NewPvcCloneController(
installerLabels: installerLabels,
shouldUpdateProgress: true,
},
clonerImage: clonerImage,
importerImage: importerImage,
pullPolicy: pullPolicy,
cloneSourceKind: "PersistentVolumeClaim",
tokenValidator: cc.NewCloneTokenValidator(common.CloneTokenIssuer, tokenPublicKey),
clonerImage: clonerImage,
importerImage: importerImage,
pullPolicy: pullPolicy,
cloneSourceKind: "PersistentVolumeClaim",
shortTokenValidator: cc.NewCloneTokenValidator(common.CloneTokenIssuer, tokenPublicKey),
longTokenValidator: cc.NewCloneTokenValidator(common.ExtendedCloneTokenIssuer, tokenPublicKey),
// for long term tokens to handle cross namespace dumb clones
tokenGenerator: newLongTermCloneTokenGenerator(tokenPrivateKey),
},
Expand Down Expand Up @@ -199,11 +200,18 @@ func (r *PvcCloneReconciler) cleanup(syncState *dvSyncState) error {
}

func addCloneToken(dv *cdiv1.DataVolume, pvc *corev1.PersistentVolumeClaim) error {
token, ok := dv.Annotations[cc.AnnCloneToken]
if !ok {
return errors.Errorf("no clone token")
// first clear out tokens that may have already been added
delete(pvc.Annotations, cc.AnnCloneToken)
delete(pvc.Annotations, cc.AnnExtendedCloneToken)
if isCrossNamespaceClone(dv) {
// only want this initially
// extended token is added later
token, ok := dv.Annotations[cc.AnnCloneToken]
if !ok {
return errors.Errorf("no clone token")
}
cc.AddAnnotation(pvc, cc.AnnCloneToken, token)
}
cc.AddAnnotation(pvc, cc.AnnCloneToken, token)
return nil
}

Expand Down Expand Up @@ -251,6 +259,13 @@ func (r *PvcCloneReconciler) syncClone(log logr.Logger, req reconcile.Request) (
return syncRes, nil
}

if addedToken, err := r.ensureExtendedTokenDV(datavolume); err != nil {
return syncRes, err
} else if addedToken {
// make sure token gets persisted before doing anything else
return syncRes, nil
}

if pvc == nil {
// Check if source PVC exists and do proper validation before attempting to clone
if done, err := r.validateCloneAndSourcePVC(&syncRes, log); err != nil {
Expand Down Expand Up @@ -322,7 +337,7 @@ func (r *PvcCloneReconciler) syncClone(log logr.Logger, req reconcile.Request) (
cc.AddAnnotation(datavolume, cc.AnnCloneType, string(cdiv1.CloneStrategyHostAssisted))
}

if err := r.ensureExtendedToken(pvc); err != nil {
if err := r.ensureExtendedTokenPVC(datavolume, pvc); err != nil {
return syncRes, err
}

Expand Down
23 changes: 20 additions & 3 deletions pkg/controller/datavolume/pvc-clone-controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,23 @@ var _ = Describe("All DataVolume Tests", func() {
storageClass = CreateStorageClassWithProvisioner(scName, map[string]string{AnnDefaultStorageClass: "true"}, map[string]string{}, pluginName)
})

It("should add extended token", func() {
dv := newCloneDataVolumeWithPVCNS("test-dv", "source-ns")
srcPvc := CreatePvcInStorageClass("test", "source-ns", &scName, nil, nil, corev1.ClaimBound)
reconciler = createCloneReconciler(storageClass, csiDriver, dv, srcPvc)
result, err := reconciler.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-dv", Namespace: metav1.NamespaceDefault}})
Expect(err).ToNot(HaveOccurred())
Expect(result.Requeue).To(BeFalse())
Expect(result.RequeueAfter).To(BeZero())
dv = &cdiv1.DataVolume{}
err = reconciler.client.Get(context.TODO(), types.NamespacedName{Name: "test-dv", Namespace: metav1.NamespaceDefault}, dv)
Expect(err).ToNot(HaveOccurred())
Expect(dv.Annotations).To(HaveKey(AnnExtendedCloneToken))
})

It("should add finalizer for cross namespace clone", func() {
dv := newCloneDataVolumeWithPVCNS("test-dv", "source-ns")
dv.Annotations[AnnExtendedCloneToken] = "test-token"
srcPvc := CreatePvcInStorageClass("test", "source-ns", &scName, nil, nil, corev1.ClaimBound)
reconciler = createCloneReconciler(storageClass, csiDriver, dv, srcPvc)
result, err := reconciler.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-dv", Namespace: metav1.NamespaceDefault}})
Expand All @@ -102,6 +117,7 @@ var _ = Describe("All DataVolume Tests", func() {

DescribeTable("should create PVC and VolumeCloneSource CR", func(sourceNamespace string) {
dv := newCloneDataVolume("test-dv")
dv.Annotations[AnnExtendedCloneToken] = "foobar"
dv.Spec.Source.PVC.Namespace = sourceNamespace
if sourceNamespace != dv.Namespace {
dv.Finalizers = append(dv.Finalizers, crossNamespaceFinalizer)
Expand Down Expand Up @@ -748,9 +764,10 @@ func createCloneReconcilerWithoutConfig(objects ...runtime.Object) *PvcCloneReco
},
shouldUpdateProgress: true,
},
tokenValidator: &FakeValidator{Match: "foobar"},
tokenGenerator: &FakeGenerator{token: "foobar"},
cloneSourceKind: "PersistentVolumeClaim",
shortTokenValidator: &FakeValidator{Match: "foobar"},
longTokenValidator: &FakeValidator{Match: "foobar", Params: map[string]string{"uid": "uid"}},
tokenGenerator: &FakeGenerator{token: "foobar"},
cloneSourceKind: "PersistentVolumeClaim",
},
}
return r
Expand Down
20 changes: 13 additions & 7 deletions pkg/controller/datavolume/snapshot-clone-controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ func NewSnapshotCloneController(
pullPolicy: pullPolicy,
cloneSourceAPIGroup: pointer.String(snapshotv1.GroupName),
cloneSourceKind: "VolumeSnapshot",
tokenValidator: cc.NewCloneTokenValidator(common.CloneTokenIssuer, tokenPublicKey),
shortTokenValidator: cc.NewCloneTokenValidator(common.CloneTokenIssuer, tokenPublicKey),
longTokenValidator: cc.NewCloneTokenValidator(common.ExtendedCloneTokenIssuer, tokenPublicKey),
// for long term tokens to handle cross namespace dumb clones
tokenGenerator: newLongTermCloneTokenGenerator(tokenPrivateKey),
},
Expand Down Expand Up @@ -153,15 +154,13 @@ func (r *SnapshotCloneReconciler) updateAnnotations(dataVolume *cdiv1.DataVolume
if dataVolume.Spec.Source.Snapshot == nil {
return errors.Errorf("no source set for clone datavolume")
}
if err := addCloneToken(dataVolume, pvc); err != nil {
return err
}
sourceNamespace := dataVolume.Spec.Source.Snapshot.Namespace
if sourceNamespace == "" {
sourceNamespace = dataVolume.Namespace
}
token, ok := dataVolume.Annotations[cc.AnnCloneToken]
if !ok {
return errors.Errorf("no clone token")
}
pvc.Annotations[cc.AnnCloneToken] = token
tempPvcName := getTempHostAssistedSourcePvcName(dataVolume)
pvc.Annotations[cc.AnnCloneRequest] = sourceNamespace + "/" + tempPvcName
return nil
Expand Down Expand Up @@ -193,6 +192,13 @@ func (r *SnapshotCloneReconciler) syncSnapshotClone(log logr.Logger, req reconci
return syncRes, nil
}

if addedToken, err := r.ensureExtendedTokenDV(datavolume); err != nil {
return syncRes, err
} else if addedToken {
// make sure token gets persisted before doing anything else
return syncRes, nil
}

if pvc == nil {
if datavolume.Spec.Storage != nil {
done, err := r.detectCloneSize(log, &syncRes)
Expand Down Expand Up @@ -257,7 +263,7 @@ func (r *SnapshotCloneReconciler) syncSnapshotClone(log logr.Logger, req reconci
cc.AddAnnotation(datavolume, cc.AnnCloneType, string(cdiv1.CloneStrategyHostAssisted))
}

if err := r.ensureExtendedToken(pvc); err != nil {
if err := r.ensureExtendedTokenPVC(datavolume, pvc); err != nil {
return syncRes, err
}

Expand Down
19 changes: 18 additions & 1 deletion pkg/controller/datavolume/snapshot-clone-controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,23 @@ var _ = Describe("All DataVolume Tests", func() {
storageClass = CreateStorageClassWithProvisioner(scName, map[string]string{AnnDefaultStorageClass: "true"}, map[string]string{}, pluginName)
})

It("should add extended token", func() {
dv := newCloneFromSnapshotDataVolumeWithPVCNS("test-dv", "source-ns")
snapshot := createSnapshotInVolumeSnapshotClass("test-snap", "source-ns", &expectedSnapshotClass, nil, nil, true)
reconciler = createSnapshotCloneReconciler(storageClass, csiDriver, dv, snapshot)
result, err := reconciler.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-dv", Namespace: metav1.NamespaceDefault}})
Expect(err).ToNot(HaveOccurred())
Expect(result.Requeue).To(BeFalse())
Expect(result.RequeueAfter).To(BeZero())
dv = &cdiv1.DataVolume{}
err = reconciler.client.Get(context.TODO(), types.NamespacedName{Name: "test-dv", Namespace: metav1.NamespaceDefault}, dv)
Expect(err).ToNot(HaveOccurred())
Expect(dv.Annotations).To(HaveKey(AnnExtendedCloneToken))
})

It("should add finalizer for cross namespace clone", func() {
dv := newCloneFromSnapshotDataVolumeWithPVCNS("test-dv", "source-ns")
dv.Annotations[AnnExtendedCloneToken] = "test-token"
snapshot := createSnapshotInVolumeSnapshotClass("test-snap", "source-ns", &expectedSnapshotClass, nil, nil, true)
reconciler = createSnapshotCloneReconciler(storageClass, csiDriver, dv, snapshot)
result, err := reconciler.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-dv", Namespace: metav1.NamespaceDefault}})
Expand All @@ -248,6 +263,7 @@ var _ = Describe("All DataVolume Tests", func() {

DescribeTable("should create PVC and VolumeCloneSource CR", func(sourceNamespace string) {
dv := newCloneFromSnapshotDataVolumeWithPVCNS("test-dv", sourceNamespace)
dv.Annotations[AnnExtendedCloneToken] = "foobar"
if sourceNamespace != dv.Namespace {
dv.Finalizers = append(dv.Finalizers, crossNamespaceFinalizer)
}
Expand Down Expand Up @@ -511,7 +527,8 @@ func createSnapshotCloneReconcilerWithoutConfig(objects ...runtime.Object) *Snap
},
shouldUpdateProgress: true,
},
tokenValidator: &FakeValidator{Match: "foobar"},
shortTokenValidator: &FakeValidator{Match: "foobar"},
longTokenValidator: &FakeValidator{Match: "foobar", Params: map[string]string{"uid": "uid"}},
tokenGenerator: &FakeGenerator{token: "foobar"},
cloneSourceAPIGroup: pointer.String("snapshot.storage.k8s.io"),
cloneSourceKind: "VolumeSnapshot",
Expand Down
44 changes: 23 additions & 21 deletions tests/cloner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3045,27 +3045,29 @@ func completeClone(f *framework.Framework, targetNs *v1.Namespace, targetPvc *v1
}, 90*time.Second, 2*time.Second).Should(BeTrue())
}
case "copy":
s, err := f.K8sClient.CoreV1().Secrets(f.CdiInstallNs).Get(context.TODO(), "cdi-api-signing-key", metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
bytes, ok := s.Data["id_rsa.pub"]
Expect(ok).To(BeTrue())
objs, err := cert.ParsePublicKeysPEM(bytes)
Expect(err).ToNot(HaveOccurred())
Expect(objs).To(HaveLen(1))
v := token.NewValidator("cdi-deployment", objs[0].(*rsa.PublicKey), time.Minute)

By("checking long token added")
Eventually(func(g Gomega) bool {
pvc, err := f.K8sClient.CoreV1().PersistentVolumeClaims(targetNs.Name).Get(context.TODO(), targetPvc.Name, metav1.GetOptions{})
g.Expect(err).ToNot(HaveOccurred())
t, ok := pvc.Annotations[controller.AnnExtendedCloneToken]
if !ok {
return false
}
_, err = v.Validate(t)
g.Expect(err).ToNot(HaveOccurred())
return true
}, 10*time.Second, assertionPollInterval).Should(BeTrue())
if sns != dv.Namespace {
s, err := f.K8sClient.CoreV1().Secrets(f.CdiInstallNs).Get(context.TODO(), "cdi-api-signing-key", metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
bytes, ok := s.Data["id_rsa.pub"]
Expect(ok).To(BeTrue())
objs, err := cert.ParsePublicKeysPEM(bytes)
Expect(err).ToNot(HaveOccurred())
Expect(objs).To(HaveLen(1))
v := token.NewValidator("cdi-deployment", objs[0].(*rsa.PublicKey), time.Minute)

By("checking long token added")
Eventually(func(g Gomega) bool {
pvc, err := f.K8sClient.CoreV1().PersistentVolumeClaims(targetNs.Name).Get(context.TODO(), targetPvc.Name, metav1.GetOptions{})
g.Expect(err).ToNot(HaveOccurred())
t, ok := pvc.Annotations[controller.AnnExtendedCloneToken]
if !ok {
return false
}
_, err = v.Validate(t)
g.Expect(err).ToNot(HaveOccurred())
return true
}, 10*time.Second, assertionPollInterval).Should(BeTrue())
}
}
}

Expand Down

0 comments on commit 75819aa

Please sign in to comment.