From 3c419396741556c1f34951dee93861255fb5de6b Mon Sep 17 00:00:00 2001 From: Rajat Gupta <37516416+r4rajat@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:09:51 +0530 Subject: [PATCH] New Parameter *insecureTLS* for kanister functions using restic (#2589) * Update Dockerfiles to use latest version of the restic to support --insecure-tls flag Signed-off-by: Rajat Gupta * Update restic wrappers to support insecureTLS Flag Signed-off-by: Rajat Gupta * Update function BackupData for insecureTLS support Signed-off-by: Rajat Gupta * Update function BackupDataAll for insecureTLS support Signed-off-by: Rajat Gupta * Update function CheckRepository for insecureTLS support Signed-off-by: Rajat Gupta * Update function CopyVolumeData for insecureTLS support Signed-off-by: Rajat Gupta * Update function DeleteData for insecureTLS support Signed-off-by: Rajat Gupta * Update function DeleteDataAll for insecureTLS support Signed-off-by: Rajat Gupta * Update function RestoreData for insecureTLS support Signed-off-by: Rajat Gupta * Update function RestoreDataAll for insecureTLS support Signed-off-by: Rajat Gupta * Update tests for Restore Data Signed-off-by: Rajat Gupta * Update Documentation for Kanister Functions (#2610) Signed-off-by: Rajat Gupta --------- Signed-off-by: Rajat Gupta Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- .../image/Dockerfile | 2 +- docs/functions.rst | 7 +++ pkg/function/backup_data.go | 15 ++++-- pkg/function/backup_data_all.go | 12 +++-- pkg/function/checkRepository.go | 14 +++-- pkg/function/copy_volume_data.go | 14 +++-- pkg/function/delete_data.go | 20 ++++--- pkg/function/delete_data_all.go | 7 ++- pkg/function/restore_data.go | 40 ++++++++------ pkg/function/restore_data_all.go | 24 +++++---- pkg/function/restore_data_test.go | 2 +- pkg/restic/restic.go | 54 +++++++++++++------ 12 files changed, 148 insertions(+), 63 deletions(-) diff --git a/docker/kanister-mongodb-replicaset/image/Dockerfile b/docker/kanister-mongodb-replicaset/image/Dockerfile index 527ee47e35..f1b333e38b 100755 --- a/docker/kanister-mongodb-replicaset/image/Dockerfile +++ b/docker/kanister-mongodb-replicaset/image/Dockerfile @@ -7,6 +7,6 @@ ADD . /kanister RUN /kanister/install.sh && rm -rf /kanister && rm -rf /tmp && mkdir /tmp -COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic +COPY --from=restic/restic:0.16.2 /usr/bin/restic /usr/local/bin/restic CMD ["tail", "-f", "/dev/null"] diff --git a/docs/functions.rst b/docs/functions.rst index 23cb82941a..5404bd5de4 100644 --- a/docs/functions.rst +++ b/docs/functions.rst @@ -306,6 +306,7 @@ Arguments: `includePath`, Yes, `string`, path of the data to be backed up `backupArtifactPrefix`, Yes, `string`, path to store the backup on the object store `encryptionKey`, No, `string`, encryption key to be used for backups + `insecureTLS`, No, `bool`, enables insecure connection for data mover Outputs: @@ -368,6 +369,7 @@ Arguments: `includePath`, Yes, `string`, path of the data to be backed up `backupArtifactPrefix`, Yes, `string`, path to store the backup on the object store appended by pod name later `encryptionKey`, No, `string`, encryption key to be used for backups + `insecureTLS`, No, `bool`, enables insecure connection for data mover Outputs: @@ -430,6 +432,7 @@ and restores data to the specified path. `pod`, No, `string`, pod to which the volumes are attached `volumes`, No, `map[string]string`, Mapping of `pvcName` to `mountPath` under which the volume will be available `encryptionKey`, No, `string`, encryption key to be used during backups + `insecureTLS`, No, `bool`, enables insecure connection for data mover `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with .. note:: @@ -505,6 +508,7 @@ respective PVCs and restores data to the specified path. `pods`, No, `string`, pods to which the volumes are attached `encryptionKey`, No, `string`, encryption key to be used during backups `backupInfo`, Yes, `string`, snapshot info generated as output in BackupDataAll function + `insecureTLS`, No, `bool`, enables insecure connection for data mover `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with .. note:: @@ -575,6 +579,7 @@ Arguments: `volume`, Yes, `string`, name of the source PVC `dataArtifactPrefix`, Yes, `string`, path on the object store to store the data in `encryptionKey`, No, `string`, encryption key to be used during backups + `insecureTLS`, No, `bool`, enables insecure connection for data mover `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with Outputs: @@ -620,6 +625,7 @@ This function deletes the snapshot data backed up by the :ref:`backupdata` funct `backupID`, No, `string`, (required if backupTag not provided) unique snapshot id generated during backup `backupTag`, No, `string`, (required if backupID not provided) unique tag added during the backup `encryptionKey`, No, `string`, encryption key to be used during backups + `insecureTLS`, No, `bool`, enables insecure connection for data mover `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with Example: @@ -657,6 +663,7 @@ BackupDataAll function. `backupInfo`, Yes, `string`, snapshot info generated as output in BackupDataAll function `encryptionKey`, No, `string`, encryption key to be used during backups `reclaimSpace`, No, `bool`, provides a way to specify if space should be reclaimed + `insecureTLS`, No, `bool`, enables insecure connection for data mover `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with Example: diff --git a/pkg/function/backup_data.go b/pkg/function/backup_data.go index 8dc2396cd4..5cd791328a 100644 --- a/pkg/function/backup_data.go +++ b/pkg/function/backup_data.go @@ -60,6 +60,8 @@ const ( BackupDataOutputBackupSize = "size" // BackupDataOutputBackupPhysicalSize is the key used for returning physical size taken by the snapshot BackupDataOutputBackupPhysicalSize = "phySize" + // InsecureTLS is the key name which provides an option to a user to disable tls + InsecureTLS = "insecureTLS" ) func init() { @@ -83,6 +85,7 @@ func (b *backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args var namespace, pod, container, includePath, backupArtifactPrefix, encryptionKey string var err error + var insecureTLS bool if err = Arg(args, BackupDataNamespaceArg, &namespace); err != nil { return nil, err } @@ -101,6 +104,9 @@ func (b *backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args if err = OptArg(args, BackupDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { return nil, err } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return nil, err + } if err = ValidateProfile(tp.Profile); err != nil { return nil, errors.Wrapf(err, "Failed to validate Profile") @@ -114,7 +120,7 @@ func (b *backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args } ctx = field.Context(ctx, consts.PodNameKey, pod) ctx = field.Context(ctx, consts.ContainerNameKey, container) - backupOutputs, err := backupData(ctx, cli, namespace, pod, container, backupArtifactPrefix, includePath, encryptionKey, tp) + backupOutputs, err := backupData(ctx, cli, namespace, pod, container, backupArtifactPrefix, includePath, encryptionKey, insecureTLS, tp) if err != nil { return nil, errors.Wrapf(err, "Failed to backup data") } @@ -147,6 +153,7 @@ func (*backupDataFunc) Arguments() []string { BackupDataIncludePathArg, BackupDataBackupArtifactPrefixArg, BackupDataEncryptionKeyArg, + InsecureTLS, } } @@ -158,19 +165,19 @@ type backupDataParsedOutput struct { phySize string } -func backupData(ctx context.Context, cli kubernetes.Interface, namespace, pod, container, backupArtifactPrefix, includePath, encryptionKey string, tp param.TemplateParams) (backupDataParsedOutput, error) { +func backupData(ctx context.Context, cli kubernetes.Interface, namespace, pod, container, backupArtifactPrefix, includePath, encryptionKey string, insecureTLS bool, tp param.TemplateParams) (backupDataParsedOutput, error) { pw, err := GetPodWriter(cli, ctx, namespace, pod, container, tp.Profile) if err != nil { return backupDataParsedOutput{}, err } defer CleanUpCredsFile(ctx, pw, namespace, pod, container) - if err = restic.GetOrCreateRepository(cli, namespace, pod, container, backupArtifactPrefix, encryptionKey, tp.Profile); err != nil { + if err = restic.GetOrCreateRepository(cli, namespace, pod, container, backupArtifactPrefix, encryptionKey, insecureTLS, tp.Profile); err != nil { return backupDataParsedOutput{}, err } // Create backup and dump it on the object store backupTag := rand.String(10) - cmd, err := restic.BackupCommandByTag(tp.Profile, backupArtifactPrefix, backupTag, includePath, encryptionKey) + cmd, err := restic.BackupCommandByTag(tp.Profile, backupArtifactPrefix, backupTag, includePath, encryptionKey, insecureTLS) if err != nil { return backupDataParsedOutput{}, err } diff --git a/pkg/function/backup_data_all.go b/pkg/function/backup_data_all.go index 00b56c91dd..a52328ea47 100644 --- a/pkg/function/backup_data_all.go +++ b/pkg/function/backup_data_all.go @@ -81,6 +81,7 @@ func (b *backupDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, a var namespace, pods, container, includePath, backupArtifactPrefix, encryptionKey string var err error + var insecureTLS bool if err = Arg(args, BackupDataAllNamespaceArg, &namespace); err != nil { return nil, err } @@ -99,6 +100,9 @@ func (b *backupDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, a if err = OptArg(args, BackupDataAllEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { return nil, err } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return nil, err + } if err = ValidateProfile(tp.Profile); err != nil { return nil, errors.Wrapf(err, "Failed to validate Profile") @@ -124,7 +128,7 @@ func (b *backupDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, a ps = strings.Fields(pods) } ctx = field.Context(ctx, consts.ContainerNameKey, container) - return backupDataAll(ctx, cli, namespace, ps, container, backupArtifactPrefix, includePath, encryptionKey, tp) + return backupDataAll(ctx, cli, namespace, ps, container, backupArtifactPrefix, includePath, encryptionKey, insecureTLS, tp) } func (*backupDataAllFunc) RequiredArgs() []string { @@ -144,10 +148,12 @@ func (*backupDataAllFunc) Arguments() []string { BackupDataAllBackupArtifactPrefixArg, BackupDataAllPodsArg, BackupDataAllEncryptionKeyArg, + InsecureTLS, } } -func backupDataAll(ctx context.Context, cli kubernetes.Interface, namespace string, ps []string, container string, backupArtifactPrefix, includePath, encryptionKey string, tp param.TemplateParams) (map[string]interface{}, error) { +func backupDataAll(ctx context.Context, cli kubernetes.Interface, namespace string, ps []string, container string, backupArtifactPrefix, includePath, encryptionKey string, + insecureTLS bool, tp param.TemplateParams) (map[string]interface{}, error) { errChan := make(chan error, len(ps)) outChan := make(chan BackupInfo, len(ps)) Output := make(map[string]BackupInfo) @@ -155,7 +161,7 @@ func backupDataAll(ctx context.Context, cli kubernetes.Interface, namespace stri for _, pod := range ps { go func(pod string, container string) { ctx = field.Context(ctx, consts.PodNameKey, pod) - backupOutputs, err := backupData(ctx, cli, namespace, pod, container, fmt.Sprintf("%s/%s", backupArtifactPrefix, pod), includePath, encryptionKey, tp) + backupOutputs, err := backupData(ctx, cli, namespace, pod, container, fmt.Sprintf("%s/%s", backupArtifactPrefix, pod), includePath, encryptionKey, insecureTLS, tp) errChan <- errors.Wrapf(err, "Failed to backup data for pod %s", pod) outChan <- BackupInfo{PodName: pod, BackupID: backupOutputs.backupID, BackupTag: backupOutputs.backupTag} }(pod, container) diff --git a/pkg/function/checkRepository.go b/pkg/function/checkRepository.go index a240f20e15..029b4573e6 100644 --- a/pkg/function/checkRepository.go +++ b/pkg/function/checkRepository.go @@ -46,7 +46,7 @@ func (*CheckRepositoryFunc) Name() string { return CheckRepositoryFuncName } -func CheckRepository(ctx context.Context, cli kubernetes.Interface, tp param.TemplateParams, encryptionKey, targetPaths, jobPrefix string, podOverride crv1alpha1.JSONMap) (map[string]interface{}, error) { +func CheckRepository(ctx context.Context, cli kubernetes.Interface, tp param.TemplateParams, encryptionKey, targetPaths, jobPrefix string, insecureTLS bool, podOverride crv1alpha1.JSONMap) (map[string]interface{}, error) { namespace, err := kube.GetControllerNamespace() if err != nil { return nil, errors.Wrapf(err, "Failed to get controller namespace") @@ -59,7 +59,7 @@ func CheckRepository(ctx context.Context, cli kubernetes.Interface, tp param.Tem PodOverride: podOverride, } pr := kube.NewPodRunner(cli, options) - podFunc := CheckRepositoryPodFunc(cli, tp, encryptionKey, targetPaths) + podFunc := CheckRepositoryPodFunc(cli, tp, encryptionKey, targetPaths, insecureTLS) return pr.Run(ctx, podFunc) } @@ -68,6 +68,7 @@ func CheckRepositoryPodFunc( tp param.TemplateParams, encryptionKey, targetPath string, + insecureTLS bool, ) func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { return func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { pod := pc.Pod() @@ -89,6 +90,7 @@ func CheckRepositoryPodFunc( tp.Profile, targetPath, encryptionKey, + insecureTLS, cli, pod.Namespace, pod.Name, @@ -126,12 +128,17 @@ func (c *CheckRepositoryFunc) Exec(ctx context.Context, tp param.TemplateParams, defer func() { c.progressPercent = progress.CompletedPercent }() var checkRepositoryArtifactPrefix, encryptionKey string + var insecureTLS bool if err := Arg(args, CheckRepositoryArtifactPrefixArg, &checkRepositoryArtifactPrefix); err != nil { return nil, err } if err := OptArg(args, CheckRepositoryEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { return nil, err } + if err := OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return nil, err + } + podOverride, err := GetPodSpecOverride(tp, args, CheckRepositoryPodOverrideArg) if err != nil { return nil, err @@ -147,7 +154,7 @@ func (c *CheckRepositoryFunc) Exec(ctx context.Context, tp param.TemplateParams, if err != nil { return nil, errors.Wrapf(err, "Failed to create Kubernetes client") } - return CheckRepository(ctx, cli, tp, encryptionKey, checkRepositoryArtifactPrefix, CheckRepositoryJobPrefix, podOverride) + return CheckRepository(ctx, cli, tp, encryptionKey, checkRepositoryArtifactPrefix, CheckRepositoryJobPrefix, insecureTLS, podOverride) } func (*CheckRepositoryFunc) RequiredArgs() []string { @@ -158,6 +165,7 @@ func (*CheckRepositoryFunc) Arguments() []string { return []string{ CheckRepositoryArtifactPrefixArg, CheckRepositoryEncryptionKeyArg, + InsecureTLS, } } func (c *CheckRepositoryFunc) ExecutionProgress() (crv1alpha1.PhaseProgress, error) { diff --git a/pkg/function/copy_volume_data.go b/pkg/function/copy_volume_data.go index ee263a451c..9e7dd631b3 100644 --- a/pkg/function/copy_volume_data.go +++ b/pkg/function/copy_volume_data.go @@ -77,6 +77,7 @@ func copyVolumeData( pvcName, targetPath, encryptionKey string, + insecureTLS bool, podOverride map[string]interface{}, ) (map[string]interface{}, error) { // Validate PVC exists @@ -99,7 +100,7 @@ func copyVolumeData( PodOverride: podOverride, } pr := kube.NewPodRunner(cli, options) - podFunc := copyVolumeDataPodFunc(cli, tp, mountPoint, targetPath, encryptionKey) + podFunc := copyVolumeDataPodFunc(cli, tp, mountPoint, targetPath, encryptionKey, insecureTLS) return pr.Run(ctx, podFunc) } @@ -109,6 +110,7 @@ func copyVolumeDataPodFunc( mountPoint, targetPath, encryptionKey string, + insecureTLS bool, ) func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { return func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { // Wait for pod to reach running state @@ -133,13 +135,14 @@ func copyVolumeDataPodFunc( pod.Spec.Containers[0].Name, targetPath, encryptionKey, + insecureTLS, tp.Profile, ); err != nil { return nil, err } // Copy data to object store backupTag := rand.String(10) - cmd, err := restic.BackupCommandByTag(tp.Profile, targetPath, backupTag, mountPoint, encryptionKey) + cmd, err := restic.BackupCommandByTag(tp.Profile, targetPath, backupTag, mountPoint, encryptionKey, insecureTLS) if err != nil { return nil, err } @@ -184,6 +187,7 @@ func (c *copyVolumeDataFunc) Exec(ctx context.Context, tp param.TemplateParams, var namespace, vol, targetPath, encryptionKey string var err error + var insecureTLS bool if err = Arg(args, CopyVolumeDataNamespaceArg, &namespace); err != nil { return nil, err } @@ -196,6 +200,9 @@ func (c *copyVolumeDataFunc) Exec(ctx context.Context, tp param.TemplateParams, if err = OptArg(args, CopyVolumeDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { return nil, err } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return nil, err + } podOverride, err := GetPodSpecOverride(tp, args, CopyVolumeDataPodOverrideArg) if err != nil { return nil, err @@ -211,7 +218,7 @@ func (c *copyVolumeDataFunc) Exec(ctx context.Context, tp param.TemplateParams, if err != nil { return nil, errors.Wrapf(err, "Failed to create Kubernetes client") } - return copyVolumeData(ctx, cli, tp, namespace, vol, targetPath, encryptionKey, podOverride) + return copyVolumeData(ctx, cli, tp, namespace, vol, targetPath, encryptionKey, insecureTLS, podOverride) } func (*copyVolumeDataFunc) RequiredArgs() []string { @@ -228,6 +235,7 @@ func (*copyVolumeDataFunc) Arguments() []string { CopyVolumeDataVolumeArg, CopyVolumeDataArtifactPrefixArg, CopyVolumeDataEncryptionKeyArg, + InsecureTLS, } } diff --git a/pkg/function/delete_data.go b/pkg/function/delete_data.go index 6516673f31..61df99cd32 100644 --- a/pkg/function/delete_data.go +++ b/pkg/function/delete_data.go @@ -79,6 +79,7 @@ func deleteData( reclaimSpace bool, namespace, encryptionKey string, + insecureTLS bool, targetPaths, deleteTags, deleteIdentifiers []string, @@ -97,7 +98,7 @@ func deleteData( PodOverride: podOverride, } pr := kube.NewPodRunner(cli, options) - podFunc := deleteDataPodFunc(tp, reclaimSpace, encryptionKey, targetPaths, deleteTags, deleteIdentifiers) + podFunc := deleteDataPodFunc(tp, reclaimSpace, encryptionKey, insecureTLS, targetPaths, deleteTags, deleteIdentifiers) return pr.Run(ctx, podFunc) } @@ -106,6 +107,7 @@ func deleteDataPodFunc( tp param.TemplateParams, reclaimSpace bool, encryptionKey string, + insecureTLS bool, targetPaths, deleteTags, deleteIdentifiers []string, @@ -133,7 +135,7 @@ func deleteDataPodFunc( } for i, deleteTag := range deleteTags { - cmd, err := restic.SnapshotsCommandByTag(tp.Profile, targetPaths[i], deleteTag, encryptionKey) + cmd, err := restic.SnapshotsCommandByTag(tp.Profile, targetPaths[i], deleteTag, encryptionKey, insecureTLS) if err != nil { return nil, err } @@ -153,7 +155,7 @@ func deleteDataPodFunc( } var spaceFreedTotal int64 for i, deleteIdentifier := range deleteIdentifiers { - cmd, err := restic.ForgetCommandByID(tp.Profile, targetPaths[i], deleteIdentifier, encryptionKey) + cmd, err := restic.ForgetCommandByID(tp.Profile, targetPaths[i], deleteIdentifier, encryptionKey, insecureTLS) if err != nil { return nil, err } @@ -166,7 +168,7 @@ func deleteDataPodFunc( return nil, errors.Wrapf(err, "Failed to forget data") } if reclaimSpace { - spaceFreedStr, err := pruneData(tp, pod, podCommandExecutor, encryptionKey, targetPaths[i]) + spaceFreedStr, err := pruneData(tp, pod, podCommandExecutor, encryptionKey, targetPaths[i], insecureTLS) if err != nil { return nil, errors.Wrapf(err, "Error executing prune command") } @@ -186,8 +188,9 @@ func pruneData( podCommandExecutor kube.PodCommandExecutor, encryptionKey, targetPath string, + insecureTLS bool, ) (string, error) { - cmd, err := restic.PruneCommand(tp.Profile, targetPath, encryptionKey) + cmd, err := restic.PruneCommand(tp.Profile, targetPath, encryptionKey, insecureTLS) if err != nil { return "", err } @@ -209,6 +212,7 @@ func (d *deleteDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args var namespace, deleteArtifactPrefix, deleteIdentifier, deleteTag, encryptionKey string var reclaimSpace bool var err error + var insecureTLS bool if err = Arg(args, DeleteDataNamespaceArg, &namespace); err != nil { return nil, err } @@ -227,6 +231,9 @@ func (d *deleteDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args if err = OptArg(args, DeleteDataReclaimSpace, &reclaimSpace, false); err != nil { return nil, err } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return nil, err + } podOverride, err := GetPodSpecOverride(tp, args, DeleteDataPodOverrideArg) if err != nil { return nil, err @@ -242,7 +249,7 @@ func (d *deleteDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args if err != nil { return nil, errors.Wrapf(err, "Failed to create Kubernetes client") } - return deleteData(ctx, cli, tp, reclaimSpace, namespace, encryptionKey, strings.Fields(deleteArtifactPrefix), strings.Fields(deleteTag), strings.Fields(deleteIdentifier), deleteDataJobPrefix, podOverride) + return deleteData(ctx, cli, tp, reclaimSpace, namespace, encryptionKey, insecureTLS, strings.Fields(deleteArtifactPrefix), strings.Fields(deleteTag), strings.Fields(deleteIdentifier), deleteDataJobPrefix, podOverride) } func (*deleteDataFunc) RequiredArgs() []string { @@ -260,6 +267,7 @@ func (*deleteDataFunc) Arguments() []string { DeleteDataBackupTagArg, DeleteDataEncryptionKeyArg, DeleteDataReclaimSpace, + InsecureTLS, } } diff --git a/pkg/function/delete_data_all.go b/pkg/function/delete_data_all.go index 1e631e5bd1..51366e73cb 100644 --- a/pkg/function/delete_data_all.go +++ b/pkg/function/delete_data_all.go @@ -71,6 +71,7 @@ func (d *deleteDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, a var namespace, deleteArtifactPrefix, backupInfo, encryptionKey string var reclaimSpace bool var err error + var insecureTLS bool if err = Arg(args, DeleteDataAllNamespaceArg, &namespace); err != nil { return nil, err } @@ -86,6 +87,9 @@ func (d *deleteDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, a if err = OptArg(args, DeleteDataAllReclaimSpace, &reclaimSpace, false); err != nil { return nil, err } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return nil, err + } podOverride, err := GetPodSpecOverride(tp, args, DeleteDataAllPodOverrideArg) if err != nil { return nil, err @@ -110,7 +114,7 @@ func (d *deleteDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, a deleteIdentifiers = append(deleteIdentifiers, info.BackupID) } - return deleteData(ctx, cli, tp, reclaimSpace, namespace, encryptionKey, targetPaths, nil, deleteIdentifiers, deleteDataAllJobPrefix, podOverride) + return deleteData(ctx, cli, tp, reclaimSpace, namespace, encryptionKey, insecureTLS, targetPaths, nil, deleteIdentifiers, deleteDataAllJobPrefix, podOverride) } func (*deleteDataAllFunc) RequiredArgs() []string { @@ -128,6 +132,7 @@ func (*deleteDataAllFunc) Arguments() []string { DeleteDataAllBackupInfo, DeleteDataAllEncryptionKeyArg, DeleteDataAllReclaimSpace, + InsecureTLS, } } diff --git a/pkg/function/restore_data.go b/pkg/function/restore_data.go index 0c8a8e9318..564776ab09 100644 --- a/pkg/function/restore_data.go +++ b/pkg/function/restore_data.go @@ -72,48 +72,52 @@ func (*restoreDataFunc) Name() string { return RestoreDataFuncName } -func validateAndGetOptArgs(args map[string]interface{}, tp param.TemplateParams) (string, string, string, map[string]string, string, string, crv1alpha1.JSONMap, error) { +func validateAndGetOptArgs(args map[string]interface{}, tp param.TemplateParams) (string, string, string, map[string]string, string, string, bool, crv1alpha1.JSONMap, error) { var restorePath, encryptionKey, pod, tag, id string var vols map[string]string var podOverride crv1alpha1.JSONMap var err error + var insecureTLS bool if err = OptArg(args, RestoreDataRestorePathArg, &restorePath, "/"); err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } if err = OptArg(args, RestoreDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } if err = OptArg(args, RestoreDataPodArg, &pod, ""); err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } if err = OptArg(args, RestoreDataVolsArg, &vols, nil); err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } if (pod != "") == (len(vols) > 0) { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, errors.Errorf("Require one argument: %s or %s", RestoreDataPodArg, RestoreDataVolsArg) } if err = OptArg(args, RestoreDataBackupTagArg, &tag, nil); err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } if err = OptArg(args, RestoreDataBackupIdentifierArg, &id, nil); err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err + } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } if (tag != "") == (id != "") { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, errors.Errorf("Require one argument: %s or %s", RestoreDataBackupTagArg, RestoreDataBackupIdentifierArg) } podOverride, err = GetPodSpecOverride(tp, args, RestoreDataPodOverrideArg) if err != nil { - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, err + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } - return restorePath, encryptionKey, pod, vols, tag, id, podOverride, nil + return restorePath, encryptionKey, pod, vols, tag, id, insecureTLS, podOverride, err } func restoreData(ctx context.Context, cli kubernetes.Interface, tp param.TemplateParams, namespace, encryptionKey, backupArtifactPrefix, restorePath, backupTag, backupID, jobPrefix, image string, - vols map[string]string, podOverride crv1alpha1.JSONMap) (map[string]interface{}, error) { + insecureTLS bool, vols map[string]string, podOverride crv1alpha1.JSONMap) (map[string]interface{}, error) { // Validate volumes validatedVols := make(map[string]kube.VolumeMountOptions) for pvcName, mountPoint := range vols { @@ -137,7 +141,7 @@ func restoreData(ctx context.Context, cli kubernetes.Interface, tp param.Templat PodOverride: podOverride, } pr := kube.NewPodRunner(cli, options) - podFunc := restoreDataPodFunc(tp, encryptionKey, backupArtifactPrefix, restorePath, backupTag, backupID) + podFunc := restoreDataPodFunc(tp, encryptionKey, backupArtifactPrefix, restorePath, backupTag, backupID, insecureTLS) return pr.Run(ctx, podFunc) } @@ -148,6 +152,7 @@ func restoreDataPodFunc( restorePath, backupTag, backupID string, + insecureTLS bool, ) func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { return func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { pod := pc.Pod() @@ -168,9 +173,9 @@ func restoreDataPodFunc( var cmd []string // Generate restore command based on the identifier passed if backupTag != "" { - cmd, err = restic.RestoreCommandByTag(tp.Profile, backupArtifactPrefix, backupTag, restorePath, encryptionKey) + cmd, err = restic.RestoreCommandByTag(tp.Profile, backupArtifactPrefix, backupTag, restorePath, encryptionKey, insecureTLS) } else if backupID != "" { - cmd, err = restic.RestoreCommandByID(tp.Profile, backupArtifactPrefix, backupID, restorePath, encryptionKey) + cmd, err = restic.RestoreCommandByID(tp.Profile, backupArtifactPrefix, backupID, restorePath, encryptionKey, insecureTLS) } if err != nil { return nil, err @@ -211,7 +216,7 @@ func (r *restoreDataFunc) Exec(ctx context.Context, tp param.TemplateParams, arg } // Validate and get optional arguments - restorePath, encryptionKey, pod, vols, backupTag, backupID, podOverride, err := validateAndGetOptArgs(args, tp) + restorePath, encryptionKey, pod, vols, backupTag, backupID, insecureTLS, podOverride, err := validateAndGetOptArgs(args, tp) if err != nil { return nil, err } @@ -245,7 +250,7 @@ func (r *restoreDataFunc) Exec(ctx context.Context, tp param.TemplateParams, arg if err != nil { return nil, errors.Wrapf(err, "Failed to create Kubernetes client") } - return restoreData(ctx, cli, tp, namespace, encryptionKey, backupArtifactPrefix, restorePath, backupTag, backupID, restoreDataJobPrefix, image, vols, podOverride) + return restoreData(ctx, cli, tp, namespace, encryptionKey, backupArtifactPrefix, restorePath, backupTag, backupID, restoreDataJobPrefix, image, insecureTLS, vols, podOverride) } func (*restoreDataFunc) RequiredArgs() []string { @@ -268,6 +273,7 @@ func (*restoreDataFunc) Arguments() []string { RestoreDataBackupTagArg, RestoreDataBackupIdentifierArg, RestoreDataPodOverrideArg, + InsecureTLS, } } diff --git a/pkg/function/restore_data_all.go b/pkg/function/restore_data_all.go index 31a2c626f1..2fe0d0ed52 100644 --- a/pkg/function/restore_data_all.go +++ b/pkg/function/restore_data_all.go @@ -68,24 +68,28 @@ func (*restoreDataAllFunc) Name() string { return RestoreDataAllFuncName } -func validateAndGetRestoreAllOptArgs(args map[string]interface{}, tp param.TemplateParams) (string, string, []string, crv1alpha1.JSONMap, error) { +func validateAndGetRestoreAllOptArgs(args map[string]interface{}, tp param.TemplateParams) (string, string, []string, bool, crv1alpha1.JSONMap, error) { var restorePath, encryptionKey, pods string var ps []string var podOverride crv1alpha1.JSONMap var err error + var insecureTLS bool if err = OptArg(args, RestoreDataAllRestorePathArg, &restorePath, "/"); err != nil { - return restorePath, encryptionKey, ps, podOverride, err + return restorePath, encryptionKey, ps, insecureTLS, podOverride, err } if err = OptArg(args, RestoreDataAllEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { - return restorePath, encryptionKey, ps, podOverride, err + return restorePath, encryptionKey, ps, insecureTLS, podOverride, err } if err = OptArg(args, RestoreDataAllPodsArg, &pods, ""); err != nil { - return restorePath, encryptionKey, ps, podOverride, err + return restorePath, encryptionKey, ps, insecureTLS, podOverride, err + } + if err = OptArg(args, InsecureTLS, &insecureTLS, false); err != nil { + return restorePath, encryptionKey, ps, insecureTLS, podOverride, err } podOverride, err = GetPodSpecOverride(tp, args, RestoreDataAllPodOverrideArg) if err != nil { - return restorePath, encryptionKey, ps, podOverride, err + return restorePath, encryptionKey, ps, insecureTLS, podOverride, err } if pods != "" { @@ -97,11 +101,11 @@ func validateAndGetRestoreAllOptArgs(args map[string]interface{}, tp param.Templ case tp.StatefulSet != nil: ps = tp.StatefulSet.Pods default: - return restorePath, encryptionKey, ps, podOverride, errors.New("Unsupported workload type") + return restorePath, encryptionKey, ps, insecureTLS, podOverride, errors.New("Unsupported workload type") } } - return restorePath, encryptionKey, ps, podOverride, nil + return restorePath, encryptionKey, ps, insecureTLS, podOverride, err } func (r *restoreDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) { @@ -111,6 +115,7 @@ func (r *restoreDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, var namespace, image, backupArtifactPrefix, backupInfo string var err error + if err = Arg(args, RestoreDataAllNamespaceArg, &namespace); err != nil { return nil, err } @@ -125,7 +130,7 @@ func (r *restoreDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, } // Validate and get optional arguments - restorePath, encryptionKey, pods, podOverride, err := validateAndGetRestoreAllOptArgs(args, tp) + restorePath, encryptionKey, pods, insecureTLS, podOverride, err := validateAndGetRestoreAllOptArgs(args, tp) if err != nil { return nil, err } @@ -155,7 +160,7 @@ func (r *restoreDataAllFunc) Exec(ctx context.Context, tp param.TemplateParams, outputChan <- out return } - out, err = restoreData(ctx, cli, tp, namespace, encryptionKey, fmt.Sprintf("%s/%s", backupArtifactPrefix, pod), restorePath, "", input[pod].BackupID, restoreDataAllJobPrefix, image, vols, podOverride) + out, err = restoreData(ctx, cli, tp, namespace, encryptionKey, fmt.Sprintf("%s/%s", backupArtifactPrefix, pod), restorePath, "", input[pod].BackupID, restoreDataAllJobPrefix, image, insecureTLS, vols, podOverride) errChan <- errors.Wrapf(err, "Failed to restore data for pod %s", pod) outputChan <- out }(pod) @@ -197,6 +202,7 @@ func (*restoreDataAllFunc) Arguments() []string { RestoreDataAllEncryptionKeyArg, RestoreDataAllPodsArg, RestoreDataAllPodOverrideArg, + InsecureTLS, } } diff --git a/pkg/function/restore_data_test.go b/pkg/function/restore_data_test.go index b9c2ba712e..d438db10d7 100644 --- a/pkg/function/restore_data_test.go +++ b/pkg/function/restore_data_test.go @@ -110,7 +110,7 @@ func (s *RestoreDataTestSuite) TestValidateAndGetOptArgs(c *C) { }, } for _, tc := range testCases { - _, _, _, _, _, _, _, err := validateAndGetOptArgs(tc.args, tc.tp) + _, _, _, _, _, _, _, _, err := validateAndGetOptArgs(tc.args, tc.tp) c.Check(err, tc.errChecker, Commentf("Case %s failed", tc.name)) } } diff --git a/pkg/restic/restic.go b/pkg/restic/restic.go index 774dbc2e5a..b1a84ff583 100644 --- a/pkg/restic/restic.go +++ b/pkg/restic/restic.go @@ -47,34 +47,43 @@ func shCommand(command string) []string { } // BackupCommandByTag returns restic backup command with tag -func BackupCommandByTag(profile *param.Profile, repository, backupTag, includePath, encryptionKey string) ([]string, error) { +func BackupCommandByTag(profile *param.Profile, repository, backupTag, includePath, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "backup", "--tag", backupTag, includePath) + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } // RestoreCommandByID returns restic restore command with snapshotID as the identifier -func RestoreCommandByID(profile *param.Profile, repository, id, restorePath, encryptionKey string) ([]string, error) { +func RestoreCommandByID(profile *param.Profile, repository, id, restorePath, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "restore", id, "--target", restorePath) + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } // RestoreCommandByTag returns restic restore command with tag as the identifier -func RestoreCommandByTag(profile *param.Profile, repository, tag, restorePath, encryptionKey string) ([]string, error) { +func RestoreCommandByTag(profile *param.Profile, repository, tag, restorePath, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "restore", "--tag", tag, "latest", "--target", restorePath) + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } @@ -91,56 +100,71 @@ func SnapshotsCommand(profile *param.Profile, repository, encryptionKey string) } // LatestSnapshotsCommand returns restic snapshots command for last snapshots -func LatestSnapshotsCommand(profile *param.Profile, repository, encryptionKey string) ([]string, error) { +func LatestSnapshotsCommand(profile *param.Profile, repository, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "snapshots", "--last", "--json") + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } // SnapshotsCommandByTag returns restic snapshots command -func SnapshotsCommandByTag(profile *param.Profile, repository, tag, encryptionKey string) ([]string, error) { +func SnapshotsCommandByTag(profile *param.Profile, repository, tag, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "snapshots", "--tag", tag, "--json") + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } // InitCommand returns restic init command -func InitCommand(profile *param.Profile, repository, encryptionKey string) ([]string, error) { +func InitCommand(profile *param.Profile, repository, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "init") + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } // ForgetCommandByID returns restic forget command -func ForgetCommandByID(profile *param.Profile, repository, id, encryptionKey string) ([]string, error) { +func ForgetCommandByID(profile *param.Profile, repository, id, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "forget", id) + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } // PruneCommand returns restic prune command -func PruneCommand(profile *param.Profile, repository, encryptionKey string) ([]string, error) { +func PruneCommand(profile *param.Profile, repository, encryptionKey string, insecureTLS bool) ([]string, error) { cmd, err := resticArgs(profile, repository, encryptionKey) if err != nil { return nil, err } cmd = append(cmd, "prune") + if insecureTLS { + cmd = append(cmd, "--insecure-tls") + } command := strings.Join(cmd, " ") return shCommand(command), nil } @@ -259,13 +283,13 @@ func resticAzureArgs(profile *param.Profile, repository string) ([]string, error } // GetOrCreateRepository will check if the repository already exists and initialize one if not -func GetOrCreateRepository(cli kubernetes.Interface, namespace, pod, container, artifactPrefix, encryptionKey string, profile *param.Profile) error { - _, _, err := getLatestSnapshots(profile, artifactPrefix, encryptionKey, cli, namespace, pod, container) +func GetOrCreateRepository(cli kubernetes.Interface, namespace, pod, container, artifactPrefix, encryptionKey string, insecureTLS bool, profile *param.Profile) error { + _, _, err := getLatestSnapshots(profile, artifactPrefix, encryptionKey, insecureTLS, cli, namespace, pod, container) if err == nil { return nil } // Create a repository - cmd, err := InitCommand(profile, artifactPrefix, encryptionKey) + cmd, err := InitCommand(profile, artifactPrefix, encryptionKey, insecureTLS) if err != nil { return errors.Wrap(err, "Failed to create init command") } @@ -276,8 +300,8 @@ func GetOrCreateRepository(cli kubernetes.Interface, namespace, pod, container, } // CheckIfRepoIsReachable checks if repo can be reached by trying to list snapshots -func CheckIfRepoIsReachable(profile *param.Profile, artifactPrefix string, encryptionKey string, cli kubernetes.Interface, namespace string, pod string, container string) error { - _, stderr, err := getLatestSnapshots(profile, artifactPrefix, encryptionKey, cli, namespace, pod, container) +func CheckIfRepoIsReachable(profile *param.Profile, artifactPrefix string, encryptionKey string, insecureTLS bool, cli kubernetes.Interface, namespace string, pod string, container string) error { + _, stderr, err := getLatestSnapshots(profile, artifactPrefix, encryptionKey, insecureTLS, cli, namespace, pod, container) if IsPasswordIncorrect(stderr) { // If password didn't work return errors.New(PasswordIncorrect) } @@ -291,9 +315,9 @@ func CheckIfRepoIsReachable(profile *param.Profile, artifactPrefix string, encry } //nolint:unparam -func getLatestSnapshots(profile *param.Profile, artifactPrefix string, encryptionKey string, cli kubernetes.Interface, namespace string, pod string, container string) (string, string, error) { +func getLatestSnapshots(profile *param.Profile, artifactPrefix string, encryptionKey string, insecureTLS bool, cli kubernetes.Interface, namespace string, pod string, container string) (string, string, error) { // Use the latest snapshots command to check if the repository exists - cmd, err := LatestSnapshotsCommand(profile, artifactPrefix, encryptionKey) + cmd, err := LatestSnapshotsCommand(profile, artifactPrefix, encryptionKey, insecureTLS) if err != nil { return "", "", errors.Wrap(err, "Failed to create snapshot command") }