Skip to content

Commit

Permalink
Add EncryptionKey arg to Backup/Restore/CopyData func (#4161)
Browse files Browse the repository at this point in the history 
* Add SecretKey arg to Backup/Restore/CopyData func

* Change SecretKey to EncryptionKey
  • Loading branch information
@pavannd1
pavannd1 authored and Ilya Kislenko committed Oct 18, 2018
1 parent 5a34237 commit 38f228e
Show file tree
Hide file tree
Showing 6 changed files with 69 additions and 28 deletions.
11 changes: 8 additions & 3 deletions pkg/function/backup_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ const (
BackupDataBackupArtifactPrefixArg = "backupArtifactPrefix"
// BackupDataBackupIdentifierArg provides a unique ID added to the artifacts
BackupDataBackupIdentifierArg = "backupIdentifier"
// BackupDataEncryptionKeyArg provides the encryption key to be used for backups
BackupDataEncryptionKeyArg = "encryptionKey"
)

func init() {
Expand All @@ -51,7 +53,7 @@ func validateProfile(profile *param.Profile) error {
}

func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
var namespace, pod, container, includePath, backupArtifactPrefix, backupIdentifier string
var namespace, pod, container, includePath, backupArtifactPrefix, backupIdentifier, encryptionKey string
var err error
if err = Arg(args, BackupDataNamespaceArg, &namespace); err != nil {
return nil, err
Expand All @@ -71,6 +73,9 @@ func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args m
if err = Arg(args, BackupDataBackupIdentifierArg, &backupIdentifier); err != nil {
return nil, err
}
if err = OptArg(args, BackupDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil {
return nil, err
}
// Validate the Profile
if err = validateProfile(tp.Profile); err != nil {
return nil, errors.Wrapf(err, "Failed to validate Profile")
Expand All @@ -80,12 +85,12 @@ func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args m
return nil, errors.Wrapf(err, "Failed to create Kubernetes client")
}

if err = restic.GetOrCreateRepository(cli, namespace, pod, container, backupArtifactPrefix, tp.Profile); err != nil {
if err = restic.GetOrCreateRepository(cli, namespace, pod, container, backupArtifactPrefix, encryptionKey, tp.Profile); err != nil {
return nil, err
}

// Create backup and dump it on the object store
cmd := restic.BackupCommand(tp.Profile, backupArtifactPrefix, backupIdentifier, includePath)
cmd := restic.BackupCommand(tp.Profile, backupArtifactPrefix, backupIdentifier, includePath, encryptionKey)
stdout, stderr, err := kube.Exec(cli, namespace, pod, container, cmd)
format.Log(pod, container, stdout)
format.Log(pod, container, stderr)
Expand Down
14 changes: 9 additions & 5 deletions pkg/function/copy_volume_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
CopyVolumeDataOutputBackupID = "backupID"
CopyVolumeDataOutputBackupRoot = "backupRoot"
CopyVolumeDataOutputBackupArtifactLocation = "backupArtifactLocation"
CopyVolumeDataEncryptionKeyArg = "encryptionKey"
)

func init() {
Expand All @@ -40,7 +41,7 @@ func (*copyVolumeDataFunc) Name() string {
return "CopyVolumeData"
}

func copyVolumeData(ctx context.Context, cli kubernetes.Interface, tp param.TemplateParams, namespace, pvc, targetPath string) (map[string]interface{}, error) {
func copyVolumeData(ctx context.Context, cli kubernetes.Interface, tp param.TemplateParams, namespace, pvc, targetPath, encryptionKey string) (map[string]interface{}, error) {
// Validate PVC exists
if _, err := cli.CoreV1().PersistentVolumeClaims(namespace).Get(pvc, metav1.GetOptions{}); err != nil {
return nil, errors.Wrapf(err, "Failed to retrieve PVC. Namespace %s, Name %s", namespace, pvc)
Expand All @@ -60,13 +61,13 @@ func copyVolumeData(ctx context.Context, cli kubernetes.Interface, tp param.Temp
defer kube.DeletePod(context.Background(), cli, pod)

// Get restic repository
if err = restic.GetOrCreateRepository(cli, namespace, pod.Name, pod.Spec.Containers[0].Name, targetPath, tp.Profile); err != nil {
if err = restic.GetOrCreateRepository(cli, namespace, pod.Name, pod.Spec.Containers[0].Name, targetPath, encryptionKey, tp.Profile); err != nil {
return nil, err
}

// Copy data to object store
backupIdentifier := rand.String(10)
cmd := restic.BackupCommand(tp.Profile, targetPath, backupIdentifier, mountPoint)
cmd := restic.BackupCommand(tp.Profile, targetPath, backupIdentifier, mountPoint, encryptionKey)
stdout, stderr, err := kube.Exec(cli, namespace, pod.Name, pod.Spec.Containers[0].Name, cmd)
format.Log(pod.Name, pod.Spec.Containers[0].Name, stdout)
format.Log(pod.Name, pod.Spec.Containers[0].Name, stderr)
Expand All @@ -82,7 +83,7 @@ func copyVolumeData(ctx context.Context, cli kubernetes.Interface, tp param.Temp
}

func (*copyVolumeDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
var namespace, vol, targetPath string
var namespace, vol, targetPath, encryptionKey string
var err error
if err = Arg(args, CopyVolumeDataNamespaceArg, &namespace); err != nil {
return nil, err
Expand All @@ -93,11 +94,14 @@ func (*copyVolumeDataFunc) Exec(ctx context.Context, tp param.TemplateParams, ar
if err = Arg(args, CopyVolumeDataArtifactPrefixArg, &targetPath); err != nil {
return nil, err
}
if err = OptArg(args, CopyVolumeDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil {
return nil, err
}
cli, err := kube.NewClient()
if err != nil {
return nil, errors.Wrapf(err, "Failed to create Kubernetes client")
}
return copyVolumeData(ctx, cli, tp, namespace, vol, targetPath)
return copyVolumeData(ctx, cli, tp, namespace, vol, targetPath, encryptionKey)
}

func (*copyVolumeDataFunc) RequiredArgs() []string {
Expand Down
26 changes: 26 additions & 0 deletions pkg/function/data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
. "gopkg.in/check.v1"

"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"

kanister "github.com/kanisterio/kanister/pkg"
Expand Down Expand Up @@ -70,6 +71,9 @@ func newRestoreDataBlueprint(pvc string) *crv1alpha1.Blueprint {
Actions: map[string]*crv1alpha1.BlueprintAction{
actionName: &crv1alpha1.BlueprintAction{
Kind: param.StatefulSetKind,
SecretNames: []string{
"backupKey",
},
Phases: []crv1alpha1.BlueprintPhase{
crv1alpha1.BlueprintPhase{
Name: "testRestore",
Expand All @@ -80,6 +84,7 @@ func newRestoreDataBlueprint(pvc string) *crv1alpha1.Blueprint {
RestoreDataBackupArtifactPrefixArg: "{{ .Profile.Location.S3Compliant.Bucket }}/{{ .Profile.Location.S3Compliant.Prefix }}",
RestoreDataRestorePathArg: "/",
RestoreDataBackupIdentifierArg: "{{ .Time }}",
RestoreDataEncryptionKeyArg: "{{ .Secrets.backupKey.Data.password | toString }}",
RestoreDataVolsArg: map[string]string{
pvc: "/mnt/data",
},
Expand Down Expand Up @@ -107,6 +112,7 @@ func newBackupDataBlueprint() *crv1alpha1.Blueprint {
BackupDataIncludePathArg: "/etc",
BackupDataBackupArtifactPrefixArg: "{{ .Profile.Location.S3Compliant.Bucket }}/{{ .Profile.Location.S3Compliant.Prefix }}",
BackupDataBackupIdentifierArg: "{{ .Time }}",
BackupDataEncryptionKeyArg: "{{ .Secrets.backupKey.Data.password | toString }}",
},
},
},
Expand All @@ -126,6 +132,19 @@ func (s *DataSuite) TestBackupRestoreData(c *C) {
pvc, err = s.cli.CoreV1().PersistentVolumeClaims(s.namespace).Create(pvc)
c.Assert(err, IsNil)

secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "secret-datatest",
Namespace: s.namespace,
},
Type: "Opaque",
StringData: map[string]string{
"password": "myPassword",
},
}
secret, err = s.cli.CoreV1().Secrets(s.namespace).Create(secret)
c.Assert(err, IsNil)

as := crv1alpha1.ActionSpec{
Object: crv1alpha1.ObjectReference{
Kind: param.StatefulSetKind,
Expand All @@ -136,6 +155,13 @@ func (s *DataSuite) TestBackupRestoreData(c *C) {
Name: testutil.TestProfileName,
Namespace: s.namespace,
},
Secrets: map[string]crv1alpha1.ObjectReference{
"backupKey": crv1alpha1.ObjectReference{
Kind: "Secret",
Name: secret.GetName(),
Namespace: s.namespace,
},
},
}

tp, err := param.New(ctx, s.cli, s.crCli, as)
Expand Down
9 changes: 7 additions & 2 deletions pkg/function/restore_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ const (
RestoreDataPodArg = "pod"
// RestoreDataVolsArg provides a map of PVC->mountPaths to be attached
RestoreDataVolsArg = "volumes"
// RestoreDataEncryptionKeyArg provides the encryption key used during backup
RestoreDataEncryptionKeyArg = "encryptionKey"
)

func init() {
Expand Down Expand Up @@ -65,7 +67,7 @@ func fetchPodVolumes(pod string, tp param.TemplateParams) (map[string]string, er
}

func (*restoreDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
var namespace, pod, image, backupArtifactPrefix, restorePath, backupIdentifier string
var namespace, pod, image, backupArtifactPrefix, restorePath, backupIdentifier, encryptionKey string
var vols map[string]string
var err error
if err = Arg(args, RestoreDataNamespaceArg, &namespace); err != nil {
Expand All @@ -89,6 +91,9 @@ func (*restoreDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args
if err = OptArg(args, RestoreDataVolsArg, &vols, nil); err != nil {
return nil, err
}
if err = OptArg(args, RestoreDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil {
return nil, err
}
if err = validateOptArgs(pod, vols); err != nil {
return nil, err
}
Expand All @@ -97,7 +102,7 @@ func (*restoreDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args
return nil, err
}
// Generate restore command
cmd := restic.RestoreCommand(tp.Profile, backupArtifactPrefix, backupIdentifier, restorePath)
cmd := restic.RestoreCommand(tp.Profile, backupArtifactPrefix, backupIdentifier, restorePath, encryptionKey)
if len(vols) == 0 {
// Fetch Volumes
vols, err = fetchPodVolumes(pod, tp)
Expand Down
34 changes: 17 additions & 17 deletions pkg/restic/restic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,50 +18,50 @@ func shCommand(command string) []string {
}

// BackupCommand returns restic backup command
func BackupCommand(profile *param.Profile, repository, id, includePath string) []string {
cmd := resticArgs(profile, repository)
func BackupCommand(profile *param.Profile, repository, id, includePath, encryptionKey string) []string {
cmd := resticArgs(profile, repository, encryptionKey)
cmd = append(cmd, "backup")
command := strings.Join(cmd, " ")
command = fmt.Sprintf("%s --tag %s %s", command, id, includePath)
return shCommand(command)
}

// RestoreCommand returns restic restore command
func RestoreCommand(profile *param.Profile, repository, id, restorePath string) []string {
cmd := resticArgs(profile, repository)
func RestoreCommand(profile *param.Profile, repository, id, restorePath, encryptionKey string) []string {
cmd := resticArgs(profile, repository, encryptionKey)
cmd = append(cmd, "restore")
command := strings.Join(cmd, " ")
command = fmt.Sprintf("%s --tag %s latest --target %s", command, id, restorePath)
return shCommand(command)
}

// SnapshotsCommand returns restic snapshots command
func SnapshotsCommand(profile *param.Profile, repository string) []string {
cmd := resticArgs(profile, repository)
func SnapshotsCommand(profile *param.Profile, repository, encryptionKey string) []string {
cmd := resticArgs(profile, repository, encryptionKey)
cmd = append(cmd, "snapshots")
command := strings.Join(cmd, " ")
return shCommand(command)
}

// InitCommand returns restic init command
func InitCommand(profile *param.Profile, repository string) []string {
cmd := resticArgs(profile, repository)
func InitCommand(profile *param.Profile, repository, encryptionKey string) []string {
cmd := resticArgs(profile, repository, encryptionKey)
cmd = append(cmd, "init")
command := strings.Join(cmd, " ")
return shCommand(command)
}

// ForgetCommand returns restic forget command
func ForgetCommand(profile *param.Profile, repository string) []string {
cmd := resticArgs(profile, repository)
func ForgetCommand(profile *param.Profile, repository, encryptionKey string) []string {
cmd := resticArgs(profile, repository, encryptionKey)
cmd = append(cmd, "forget")
command := strings.Join(cmd, " ")
return shCommand(command)
}

// PruneCommand returns restic prune command
func PruneCommand(profile *param.Profile, repository string) []string {
cmd := resticArgs(profile, repository)
func PruneCommand(profile *param.Profile, repository, encryptionKey string) []string {
cmd := resticArgs(profile, repository, encryptionKey)
cmd = append(cmd, "prune")
command := strings.Join(cmd, " ")
return shCommand(command)
Expand All @@ -74,32 +74,32 @@ const (
awsS3Endpoint = "s3.amazonaws.com"
)

func resticArgs(profile *param.Profile, repository string) []string {
func resticArgs(profile *param.Profile, repository, encryptionKey string) []string {
s3Endpoint := awsS3Endpoint
if profile.Location.S3Compliant.Endpoint != "" {
s3Endpoint = profile.Location.S3Compliant.Endpoint
}
return []string{
fmt.Sprintf("export %s=%s\n", location.AWSAccessKeyID, profile.Credential.KeyPair.ID),
fmt.Sprintf("export %s=%s\n", location.AWSSecretAccessKey, profile.Credential.KeyPair.Secret),
fmt.Sprintf("export %s=%s\n", ResticPassword, generatePassword()),
fmt.Sprintf("export %s=%s\n", ResticPassword, encryptionKey),
fmt.Sprintf("export %s=s3:%s/%s\n", ResticRepository, s3Endpoint, repository),
ResticCommand,
}
}

// GetOrCreateRepository will check if the repository already exists and initialize one if not
func GetOrCreateRepository(cli kubernetes.Interface, namespace, pod, container, artifactPrefix string, profile *param.Profile) error {
func GetOrCreateRepository(cli kubernetes.Interface, namespace, pod, container, artifactPrefix, encryptionKey string, profile *param.Profile) error {
// Use the snapshots command to check if the repository exists
cmd := SnapshotsCommand(profile, artifactPrefix)
cmd := SnapshotsCommand(profile, artifactPrefix, encryptionKey)
stdout, stderr, err := kube.Exec(cli, namespace, pod, container, cmd)
format.Log(pod, container, stdout)
format.Log(pod, container, stderr)
if err == nil {
return nil
}
// Create a repository
cmd = InitCommand(profile, artifactPrefix)
cmd = InitCommand(profile, artifactPrefix, encryptionKey)
stdout, stderr, err = kube.Exec(cli, namespace, pod, container, cmd)
format.Log(pod, container, stdout)
format.Log(pod, container, stderr)
Expand Down
3 changes: 2 additions & 1 deletion pkg/restic/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ const (
password = "testpassword"
)

func generatePassword() string {
// GeneratePassword generates a password
func GeneratePassword() string {
h := sha256.New()
h.Write([]byte(password))
return fmt.Sprintf("%x", h.Sum(nil))
Expand Down

0 comments on commit 38f228e

Please sign in to comment.