From 2397f19ea69c7b6abef0c5f8deb833742407e72b Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 7 Oct 2024 09:22:10 +0200 Subject: [PATCH 01/16] add aes encryption with cli commands --- Makefile | 2 +- api/v1/backup.pb.go | 206 +++++++++++++++--- api/v1/backup_grpc.pb.go | 42 +++- cmd/internal/backup/backup.go | 15 +- .../backup/providers/common/common.go | 12 + cmd/internal/backup/providers/contract.go | 2 +- cmd/internal/backup/providers/gcp/gcp.go | 17 +- cmd/internal/backup/providers/local/local.go | 12 +- cmd/internal/backup/providers/s3/s3.go | 12 +- cmd/internal/encryption/encryption.go | 191 ++++++++++++++++ cmd/internal/initializer/initializer.go | 23 +- cmd/internal/initializer/service.go | 26 ++- cmd/main.go | 102 ++++++++- go.sum | 2 - proto/v1/backup.proto | 9 + 15 files changed, 598 insertions(+), 75 deletions(-) create mode 100644 cmd/internal/encryption/encryption.go diff --git a/Makefile b/Makefile index 3309e6a..95507e7 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ endif kubectl --kubeconfig $(KUBECONFIG) delete -f "deploy/$(DB)-$(BACKUP_PROVIDER).yaml" || true # for idempotence kubectl --kubeconfig $(KUBECONFIG) apply -f "deploy/$(DB)-$(BACKUP_PROVIDER).yaml" # tailing - stern --kubeconfig $(KUBECONFIG) '.*' + kubectl stern --kubeconfig $(KUBECONFIG) '.*' .PHONY: kind-cluster-create kind-cluster-create: dockerimage diff --git a/api/v1/backup.pb.go b/api/v1/backup.pb.go index ab51dca..1b5a772 100644 --- a/api/v1/backup.pb.go +++ b/api/v1/backup.pb.go @@ -254,6 +254,100 @@ func (*RestoreBackupResponse) Descriptor() ([]byte, []int) { return file_v1_backup_proto_rawDescGZIP(), []int{4} } +type GetBackupByVersionRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *GetBackupByVersionRequest) Reset() { + *x = GetBackupByVersionRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_backup_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetBackupByVersionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBackupByVersionRequest) ProtoMessage() {} + +func (x *GetBackupByVersionRequest) ProtoReflect() protoreflect.Message { + mi := &file_v1_backup_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBackupByVersionRequest.ProtoReflect.Descriptor instead. +func (*GetBackupByVersionRequest) Descriptor() ([]byte, []int) { + return file_v1_backup_proto_rawDescGZIP(), []int{5} +} + +func (x *GetBackupByVersionRequest) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +type GetBackupByVersionResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Backup *Backup `protobuf:"bytes,1,opt,name=backup,proto3" json:"backup,omitempty"` +} + +func (x *GetBackupByVersionResponse) Reset() { + *x = GetBackupByVersionResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_backup_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetBackupByVersionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBackupByVersionResponse) ProtoMessage() {} + +func (x *GetBackupByVersionResponse) ProtoReflect() protoreflect.Message { + mi := &file_v1_backup_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBackupByVersionResponse.ProtoReflect.Descriptor instead. +func (*GetBackupByVersionResponse) Descriptor() ([]byte, []int) { + return file_v1_backup_proto_rawDescGZIP(), []int{6} +} + +func (x *GetBackupByVersionResponse) GetBackup() *Backup { + if x != nil { + return x.Backup + } + return nil +} + var File_v1_backup_proto protoreflect.FileDescriptor var file_v1_backup_proto_rawDesc = []byte{ @@ -277,23 +371,36 @@ var file_v1_backup_proto_rawDesc = []byte{ 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x17, 0x0a, 0x15, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x94, 0x01, 0x0a, 0x0d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x42, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x67, 0x0a, 0x06, - 0x63, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x6c, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x72, - 0x6f, 0x70, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0xa2, - 0x02, 0x03, 0x56, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x56, 0x31, 0xca, 0x02, 0x02, 0x56, 0x31, 0xe2, - 0x02, 0x0e, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0xea, 0x02, 0x02, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x35, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x40, 0x0a, 0x1a, + 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x06, 0x62, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x76, 0x31, 0x2e, + 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x32, 0xe9, + 0x01, 0x0a, 0x0d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x3d, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x12, + 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x44, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x67, 0x0a, 0x06, 0x63, 0x6f, + 0x6d, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x6d, 0x65, 0x74, 0x61, 0x6c, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x72, 0x6f, 0x70, + 0x74, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0xa2, 0x02, 0x03, + 0x56, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x56, 0x31, 0xca, 0x02, 0x02, 0x56, 0x31, 0xe2, 0x02, 0x0e, + 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x02, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -308,27 +415,32 @@ func file_v1_backup_proto_rawDescGZIP() []byte { return file_v1_backup_proto_rawDescData } -var file_v1_backup_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_v1_backup_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_v1_backup_proto_goTypes = []any{ - (*ListBackupsRequest)(nil), // 0: v1.ListBackupsRequest - (*BackupListResponse)(nil), // 1: v1.BackupListResponse - (*Backup)(nil), // 2: v1.Backup - (*RestoreBackupRequest)(nil), // 3: v1.RestoreBackupRequest - (*RestoreBackupResponse)(nil), // 4: v1.RestoreBackupResponse - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp + (*ListBackupsRequest)(nil), // 0: v1.ListBackupsRequest + (*BackupListResponse)(nil), // 1: v1.BackupListResponse + (*Backup)(nil), // 2: v1.Backup + (*RestoreBackupRequest)(nil), // 3: v1.RestoreBackupRequest + (*RestoreBackupResponse)(nil), // 4: v1.RestoreBackupResponse + (*GetBackupByVersionRequest)(nil), // 5: v1.GetBackupByVersionRequest + (*GetBackupByVersionResponse)(nil), // 6: v1.GetBackupByVersionResponse + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp } var file_v1_backup_proto_depIdxs = []int32{ 2, // 0: v1.BackupListResponse.backups:type_name -> v1.Backup - 5, // 1: v1.Backup.timestamp:type_name -> google.protobuf.Timestamp - 0, // 2: v1.BackupService.ListBackups:input_type -> v1.ListBackupsRequest - 3, // 3: v1.BackupService.RestoreBackup:input_type -> v1.RestoreBackupRequest - 1, // 4: v1.BackupService.ListBackups:output_type -> v1.BackupListResponse - 4, // 5: v1.BackupService.RestoreBackup:output_type -> v1.RestoreBackupResponse - 4, // [4:6] is the sub-list for method output_type - 2, // [2:4] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 7, // 1: v1.Backup.timestamp:type_name -> google.protobuf.Timestamp + 2, // 2: v1.GetBackupByVersionResponse.backup:type_name -> v1.Backup + 0, // 3: v1.BackupService.ListBackups:input_type -> v1.ListBackupsRequest + 3, // 4: v1.BackupService.RestoreBackup:input_type -> v1.RestoreBackupRequest + 5, // 5: v1.BackupService.GetBackupByVersion:input_type -> v1.GetBackupByVersionRequest + 1, // 6: v1.BackupService.ListBackups:output_type -> v1.BackupListResponse + 4, // 7: v1.BackupService.RestoreBackup:output_type -> v1.RestoreBackupResponse + 6, // 8: v1.BackupService.GetBackupByVersion:output_type -> v1.GetBackupByVersionResponse + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_v1_backup_proto_init() } @@ -397,6 +509,30 @@ func file_v1_backup_proto_init() { return nil } } + file_v1_backup_proto_msgTypes[5].Exporter = func(v any, i int) any { + switch v := v.(*GetBackupByVersionRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v1_backup_proto_msgTypes[6].Exporter = func(v any, i int) any { + switch v := v.(*GetBackupByVersionResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -404,7 +540,7 @@ func file_v1_backup_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v1_backup_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/api/v1/backup_grpc.pb.go b/api/v1/backup_grpc.pb.go index 36e6361..3d48371 100644 --- a/api/v1/backup_grpc.pb.go +++ b/api/v1/backup_grpc.pb.go @@ -19,8 +19,9 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - BackupService_ListBackups_FullMethodName = "/v1.BackupService/ListBackups" - BackupService_RestoreBackup_FullMethodName = "/v1.BackupService/RestoreBackup" + BackupService_ListBackups_FullMethodName = "/v1.BackupService/ListBackups" + BackupService_RestoreBackup_FullMethodName = "/v1.BackupService/RestoreBackup" + BackupService_GetBackupByVersion_FullMethodName = "/v1.BackupService/GetBackupByVersion" ) // BackupServiceClient is the client API for BackupService service. @@ -29,6 +30,7 @@ const ( type BackupServiceClient interface { ListBackups(ctx context.Context, in *ListBackupsRequest, opts ...grpc.CallOption) (*BackupListResponse, error) RestoreBackup(ctx context.Context, in *RestoreBackupRequest, opts ...grpc.CallOption) (*RestoreBackupResponse, error) + GetBackupByVersion(ctx context.Context, in *GetBackupByVersionRequest, opts ...grpc.CallOption) (*GetBackupByVersionResponse, error) } type backupServiceClient struct { @@ -59,12 +61,23 @@ func (c *backupServiceClient) RestoreBackup(ctx context.Context, in *RestoreBack return out, nil } +func (c *backupServiceClient) GetBackupByVersion(ctx context.Context, in *GetBackupByVersionRequest, opts ...grpc.CallOption) (*GetBackupByVersionResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBackupByVersionResponse) + err := c.cc.Invoke(ctx, BackupService_GetBackupByVersion_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // BackupServiceServer is the server API for BackupService service. // All implementations should embed UnimplementedBackupServiceServer // for forward compatibility. type BackupServiceServer interface { ListBackups(context.Context, *ListBackupsRequest) (*BackupListResponse, error) RestoreBackup(context.Context, *RestoreBackupRequest) (*RestoreBackupResponse, error) + GetBackupByVersion(context.Context, *GetBackupByVersionRequest) (*GetBackupByVersionResponse, error) } // UnimplementedBackupServiceServer should be embedded to have @@ -80,6 +93,9 @@ func (UnimplementedBackupServiceServer) ListBackups(context.Context, *ListBackup func (UnimplementedBackupServiceServer) RestoreBackup(context.Context, *RestoreBackupRequest) (*RestoreBackupResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RestoreBackup not implemented") } +func (UnimplementedBackupServiceServer) GetBackupByVersion(context.Context, *GetBackupByVersionRequest) (*GetBackupByVersionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetBackupByVersion not implemented") +} func (UnimplementedBackupServiceServer) testEmbeddedByValue() {} // UnsafeBackupServiceServer may be embedded to opt out of forward compatibility for this service. @@ -136,6 +152,24 @@ func _BackupService_RestoreBackup_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _BackupService_GetBackupByVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBackupByVersionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BackupServiceServer).GetBackupByVersion(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BackupService_GetBackupByVersion_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BackupServiceServer).GetBackupByVersion(ctx, req.(*GetBackupByVersionRequest)) + } + return interceptor(ctx, in, info, handler) +} + // BackupService_ServiceDesc is the grpc.ServiceDesc for BackupService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -151,6 +185,10 @@ var BackupService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RestoreBackup", Handler: _BackupService_RestoreBackup_Handler, }, + { + MethodName: "GetBackupByVersion", + Handler: _BackupService_GetBackupByVersion_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "v1/backup.proto", diff --git a/cmd/internal/backup/backup.go b/cmd/internal/backup/backup.go index cc71637..5c33061 100644 --- a/cmd/internal/backup/backup.go +++ b/cmd/internal/backup/backup.go @@ -10,6 +10,7 @@ import ( backuproviders "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" cron "github.com/robfig/cron/v3" @@ -23,6 +24,7 @@ type BackuperConfig struct { BackupProvider backuproviders.BackupProvider Metrics *metrics.Metrics Compressor *compress.Compressor + Encrypter *encryption.Encrypter } type Backuper struct { @@ -33,6 +35,7 @@ type Backuper struct { metrics *metrics.Metrics comp *compress.Compressor sem *semaphore.Weighted + encrypter *encryption.Encrypter } func New(config *BackuperConfig) *Backuper { @@ -44,7 +47,8 @@ func New(config *BackuperConfig) *Backuper { metrics: config.Metrics, comp: config.Compressor, // sem guards backups to be taken concurrently - sem: semaphore.NewWeighted(1), + sem: semaphore.NewWeighted(1), + encrypter: config.Encrypter, } } @@ -105,6 +109,15 @@ func (b *Backuper) CreateBackup(ctx context.Context) error { b.log.Info("compressed backup") + if b.encrypter != nil { + filename, err = b.encrypter.Encrypt(filename) + if err != nil { + b.metrics.CountError("encrypt") + return fmt.Errorf("error encrypting backup: %w", err) + } + b.log.Info("encrypted backup") + } + err = b.bp.UploadBackup(ctx, filename) if err != nil { b.metrics.CountError("upload") diff --git a/cmd/internal/backup/providers/common/common.go b/cmd/internal/backup/providers/common/common.go index 11917ab..a09cfcb 100644 --- a/cmd/internal/backup/providers/common/common.go +++ b/cmd/internal/backup/providers/common/common.go @@ -2,6 +2,7 @@ package common import ( "fmt" + "path/filepath" "sort" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" @@ -32,3 +33,14 @@ func Get(versions []*providers.BackupVersion, version string) (*providers.Backup } return nil, fmt.Errorf("version %q not found", version) } + +// Determines whether a filepath is specified or not (uses default downloadDir) +func DeterminBackupFilePath(outPath string, downloadDir string, fileName string) string { + backupFilePath := "" + if outPath == "" { + backupFilePath = filepath.Join(downloadDir, fileName) + } else { + backupFilePath = filepath.Join(outPath, fileName) + } + return backupFilePath +} diff --git a/cmd/internal/backup/providers/contract.go b/cmd/internal/backup/providers/contract.go index 30604d7..4740f1c 100644 --- a/cmd/internal/backup/providers/contract.go +++ b/cmd/internal/backup/providers/contract.go @@ -10,7 +10,7 @@ type BackupProvider interface { ListBackups(ctx context.Context) (BackupVersions, error) CleanupBackups(ctx context.Context) error GetNextBackupName(ctx context.Context) string - DownloadBackup(ctx context.Context, version *BackupVersion) error + DownloadBackup(ctx context.Context, version *BackupVersion, outPath string) (string, error) UploadBackup(ctx context.Context, sourcePath string) error } diff --git a/cmd/internal/backup/providers/gcp/gcp.go b/cmd/internal/backup/providers/gcp/gcp.go index 89cae60..2e1e988 100644 --- a/cmd/internal/backup/providers/gcp/gcp.go +++ b/cmd/internal/backup/providers/gcp/gcp.go @@ -6,7 +6,6 @@ import ( "io" "log/slog" "net/http" - "path" "path/filepath" "strconv" "strings" @@ -14,6 +13,7 @@ import ( "errors" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "github.com/spf13/afero" @@ -148,10 +148,10 @@ func (b *BackupProviderGCP) CleanupBackups(_ context.Context) error { } // DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *providers.BackupVersion) error { +func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outPath string) (string, error) { gen, err := strconv.ParseInt(version.Version, 10, 64) if err != nil { - return err + return "", err } bucket := b.c.Bucket(b.config.BucketName) @@ -160,28 +160,29 @@ func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *provide if strings.Contains(downloadFileName, "/") { downloadFileName = filepath.Base(downloadFileName) } - backupFilePath := path.Join(constants.DownloadDir, downloadFileName) + + backupFilePath := common.DeterminBackupFilePath(outPath, constants.DownloadDir, downloadFileName) b.log.Info("downloading", "object", version.Name, "gen", gen, "to", backupFilePath) r, err := bucket.Object(version.Name).Generation(gen).NewReader(ctx) if err != nil { - return fmt.Errorf("backup not found: %w", err) + return "", fmt.Errorf("backup not found: %w", err) } defer r.Close() f, err := b.fs.Create(backupFilePath) if err != nil { - return err + return "", err } defer f.Close() _, err = io.Copy(f, r) if err != nil { - return fmt.Errorf("error writing file from gcp to filesystem: %w", err) + return "", fmt.Errorf("error writing file from gcp to filesystem: %w", err) } - return nil + return backupFilePath, nil } // UploadBackup uploads a backup to the backup provider diff --git a/cmd/internal/backup/providers/local/local.go b/cmd/internal/backup/providers/local/local.go index 8c2a7f4..26d1e56 100644 --- a/cmd/internal/backup/providers/local/local.go +++ b/cmd/internal/backup/providers/local/local.go @@ -11,6 +11,7 @@ import ( "errors" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "github.com/spf13/afero" @@ -86,18 +87,19 @@ func (b *BackupProviderLocal) CleanupBackups(_ context.Context) error { } // DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderLocal) DownloadBackup(_ context.Context, version *providers.BackupVersion) error { +func (b *BackupProviderLocal) DownloadBackup(_ context.Context, version *providers.BackupVersion, outPath string) (string, error) { b.log.Info("download backup called for provider local") source := filepath.Join(b.config.LocalBackupPath, version.Name) - destination := filepath.Join(constants.DownloadDir, version.Name) - err := utils.Copy(b.fs, source, destination) + backupFilePath := common.DeterminBackupFilePath(outPath, constants.DownloadDir, version.Name) + + err := utils.Copy(b.fs, source, backupFilePath) if err != nil { - return err + return "", err } - return nil + return backupFilePath, err } // UploadBackup uploads a backup to the backup provider diff --git a/cmd/internal/backup/providers/s3/s3.go b/cmd/internal/backup/providers/s3/s3.go index 10200cd..7964014 100644 --- a/cmd/internal/backup/providers/s3/s3.go +++ b/cmd/internal/backup/providers/s3/s3.go @@ -3,13 +3,13 @@ package s3 import ( "context" "log/slog" - "path" "path/filepath" "strings" "errors" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "github.com/spf13/afero" @@ -190,7 +190,7 @@ func (b *BackupProviderS3) CleanupBackups(_ context.Context) error { } // DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion) error { +func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outPath string) (string, error) { bucket := aws.String(b.config.BucketName) downloadFileName := version.Name @@ -198,11 +198,11 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider downloadFileName = filepath.Base(downloadFileName) } - backupFilePath := path.Join(constants.DownloadDir, downloadFileName) + backupFilePath := common.DeterminBackupFilePath(outPath, constants.DownloadDir, downloadFileName) f, err := b.fs.Create(backupFilePath) if err != nil { - return err + return "", err } defer f.Close() @@ -217,10 +217,10 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider VersionId: &version.Version, }) if err != nil { - return err + return "", err } - return nil + return backupFilePath, nil } // UploadBackup uploads a backup to the backup provider diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go new file mode 100644 index 0000000..4665716 --- /dev/null +++ b/cmd/internal/encryption/encryption.go @@ -0,0 +1,191 @@ +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "unicode" +) + +// Suffix is appended on encryption and removed on decryption from given input +const Suffix = ".aes" + +// Encrypter is used to encrypt/decrypt backups +type Encrypter struct { + key string + log *slog.Logger +} + +// New creates a new Encrypter with the given key. +// The key should be 16 bytes (AES-128), 24 bytes (AES-192) or +// 32 bytes (AES-256) +func New(log *slog.Logger, key string) (*Encrypter, error) { + switch len(key) { + case 16, 24, 32: + default: + return nil, fmt.Errorf("key length:%d invalid, must be 16,24 or 32 bytes", len(key)) + } + if !isASCII(key) { + return nil, fmt.Errorf("key must only contain ascii characters") + } + + return &Encrypter{ + key: key, + log: log, + }, nil + +} + +func (e *Encrypter) Encrypt(input string) (string, error) { + output := input + Suffix + e.log.Info("encrypt", "input", input, "output", output) + infile, err := os.Open(input) + if err != nil { + return "", err + } + defer infile.Close() + + key := []byte(e.key) + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + // Never use more than 2^32 random nonces with a given key + // because of the risk of repeat. + iv := make([]byte, block.BlockSize()) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + + outfile, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0777) + if err != nil { + return "", err + } + defer outfile.Close() + + // The buffer size must be multiple of 16 bytes + buf := make([]byte, 1024) + stream := cipher.NewCTR(block, iv) + for { + n, err := infile.Read(buf) + if n > 0 { + stream.XORKeyStream(buf, buf[:n]) + // Write into file + _, err = outfile.Write(buf[:n]) + if err != nil { + return "", err + } + } + + if err == io.EOF { + break + } + + if err != nil { + e.log.Info("Read %d bytes: %v", strconv.Itoa(n), err) + break + } + } + // Append the IV + _, err = outfile.Write(iv) + if err == nil { + err := os.Remove(input) + if err != nil { + e.log.Warn("unable to remove input", "error", err) + } + } + return output, err +} + +// Decrypt input file with key and store decrypted result with suffix removed +// if input does not end with suffix, it is assumed that the file was not encrypted. +func (e *Encrypter) Decrypt(input string) (string, error) { + output := strings.TrimSuffix(input, Suffix) + e.log.Info("decrypt", "input", input, "output", output) + extension := filepath.Ext(input) + if extension != Suffix { + return input, fmt.Errorf("input is not encrypted") + } + infile, err := os.Open(input) + if err != nil { + return "", err + } + defer infile.Close() + + key := []byte(e.key) + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + // Never use more than 2^32 random nonces with a given key + // because of the risk of repeat. + fi, err := infile.Stat() + if err != nil { + return "", err + } + + iv := make([]byte, block.BlockSize()) + msgLen := fi.Size() - int64(len(iv)) + _, err = infile.ReadAt(iv, msgLen) + if err != nil { + return "", err + } + + outfile, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0777) + if err != nil { + return "", err + } + defer outfile.Close() + + // The buffer size must be multiple of 16 bytes + buf := make([]byte, 1024) + stream := cipher.NewCTR(block, iv) + for { + n, err := infile.Read(buf) + if n > 0 { + // The last bytes are the IV, don't belong the original message + if n > int(msgLen) { + n = int(msgLen) + } + msgLen -= int64(n) + stream.XORKeyStream(buf, buf[:n]) + // Write into file + _, err = outfile.Write(buf[:n]) + if err != nil { + return "", err + } + } + + if err == io.EOF { + break + } + + if err != nil { + e.log.Info("Read %d bytes: %v", strconv.Itoa(n), err) + break + } + } + err = os.Remove(input) + if err != nil { + e.log.Warn("unable to remove input", "error", err) + } + return output, nil +} + +func isASCII(s string) bool { + for _, c := range s { + if c > unicode.MaxASCII { + return false + } + } + return true +} diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index 135645d..08ca672 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -15,6 +15,7 @@ import ( "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" @@ -34,9 +35,10 @@ type Initializer struct { comp *compress.Compressor metrics *metrics.Metrics dbDataDir string + encrypter *encryption.Encrypter } -func New(log *slog.Logger, addr string, db database.Database, bp providers.BackupProvider, comp *compress.Compressor, metrics *metrics.Metrics, dbDataDir string) *Initializer { +func New(log *slog.Logger, addr string, db database.Database, bp providers.BackupProvider, comp *compress.Compressor, metrics *metrics.Metrics, dbDataDir string, encrypter *encryption.Encrypter) *Initializer { return &Initializer{ currentStatus: &v1.StatusResponse{ Status: v1.StatusResponse_CHECKING, @@ -49,6 +51,7 @@ func New(log *slog.Logger, addr string, db database.Database, bp providers.Backu comp: comp, dbDataDir: dbDataDir, metrics: metrics, + encrypter: encrypter, } } @@ -163,7 +166,7 @@ func (i *Initializer) initialize(ctx context.Context) error { return nil } - err = i.Restore(ctx, latestBackup) + err = i.Restore(ctx, latestBackup, false) if err != nil { return fmt.Errorf("unable to restore database: %w", err) } @@ -172,7 +175,7 @@ func (i *Initializer) initialize(ctx context.Context) error { } // Restore restores the database with the given backup version -func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVersion) error { +func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVersion, downloadOnly bool) error { i.log.Info("restoring backup", "version", version.Version, "date", version.Date.String()) i.currentStatus.Status = v1.StatusResponse_RESTORING @@ -197,17 +200,29 @@ func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVers return fmt.Errorf("could not delete priorly downloaded file: %w", err) } - err := i.bp.DownloadBackup(ctx, version) + _, err := i.bp.DownloadBackup(ctx, version, "") if err != nil { return fmt.Errorf("unable to download backup: %w", err) } + if i.encrypter != nil { + backupFilePath, err = i.encrypter.Decrypt(backupFilePath) + if err != nil { + return fmt.Errorf("unable to decrypt backup: %w", err) + } + } + i.currentStatus.Message = "uncompressing backup" err = i.comp.Decompress(backupFilePath) if err != nil { return fmt.Errorf("unable to uncompress backup: %w", err) } + if downloadOnly { + i.log.Info("downloadOnly was specified, skipping database recovery") + return nil + } + i.currentStatus.Message = "restoring backup" err = i.db.Recover(ctx) if err != nil { diff --git a/cmd/internal/initializer/service.go b/cmd/internal/initializer/service.go index df8c7b2..a0146a1 100644 --- a/cmd/internal/initializer/service.go +++ b/cmd/internal/initializer/service.go @@ -28,10 +28,10 @@ func (s *initializerService) Status(context.Context, *v1.StatusRequest) (*v1.Sta type backupService struct { bp providers.BackupProvider - restoreFn func(ctx context.Context, version *providers.BackupVersion) error + restoreFn func(ctx context.Context, version *providers.BackupVersion, downloadOnly bool) error } -func newBackupProviderService(bp providers.BackupProvider, restoreFn func(ctx context.Context, version *providers.BackupVersion) error) *backupService { +func newBackupProviderService(bp providers.BackupProvider, restoreFn func(ctx context.Context, version *providers.BackupVersion, downloadOnly bool) error) *backupService { return &backupService{ bp: bp, restoreFn: restoreFn, @@ -76,7 +76,8 @@ func (s *backupService) RestoreBackup(ctx context.Context, req *v1.RestoreBackup return nil, status.Error(codes.Internal, err.Error()) } - err = s.restoreFn(ctx, version) + //TODO -> check false value here + err = s.restoreFn(ctx, version, false) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("error restoring backup: %s", err)) } @@ -84,6 +85,25 @@ func (s *backupService) RestoreBackup(ctx context.Context, req *v1.RestoreBackup return &v1.RestoreBackupResponse{}, nil } +func (s *backupService) GetBackupByVersion(ctx context.Context, req *v1.GetBackupByVersionRequest) (*v1.GetBackupByVersionResponse, error) { + if req.GetVersion() == "" { + return nil, status.Error(codes.InvalidArgument, "version to get must be defined explicitly") + } + + versions, err := s.bp.ListBackups(ctx) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + version, err := versions.Get(req.GetVersion()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &v1.GetBackupByVersionResponse{Backup: &v1.Backup{Name: version.Name, Version: version.Version, Timestamp: timestamppb.New(version.Date)}}, nil + +} + type databaseService struct { backupFn func() error } diff --git a/cmd/main.go b/cmd/main.go index 25c2df8..ff27bed 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,6 +23,7 @@ import ( "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/postgres" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/redis" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/rethinkdb" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/initializer" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/probe" @@ -49,6 +50,7 @@ const ( databaseFlg = "db" databaseDatadirFlg = "db-data-directory" + downloadOnlyFlg = "download-only" preExecCommandsFlg = "pre-exec-cmds" postExecCommandsFlg = "post-exec-cmds" @@ -93,14 +95,17 @@ const ( s3SecretKeyFlg = "s3-secret-key" compressionMethod = "compression-method" + + encryptionKey = "encryption-key" ) var ( - cfgFile string - logger *slog.Logger - db database.Database - bp providers.BackupProvider - stop context.Context + cfgFile string + logger *slog.Logger + db database.Database + bp providers.BackupProvider + encrypter *encryption.Encrypter + stop context.Context ) var rootCmd = &cobra.Command{ @@ -124,6 +129,9 @@ var startCmd = &cobra.Command{ if err := initDatabase(); err != nil { return err } + if err := initEncrypter(); err != nil { + return err + } return initBackupProvider() }, RunE: func(cmd *cobra.Command, args []string) error { @@ -150,6 +158,18 @@ var startCmd = &cobra.Command{ metrics := metrics.New() metrics.Start(logger.WithGroup("metrics")) + var encrypter *encryption.Encrypter + key := viper.GetString(encryptionKey) + if key != "" { + encrypter, err = encryption.New(logger.WithGroup("encryption"), key) + if err != nil { + return fmt.Errorf("unable to initialize encryption:%v", err) + } + logger.Info("successfully initialized encrypter") + } else { + logger.Info("no encrypter found") + } + backuper := backup.New(&backup.BackuperConfig{ Log: logger.WithGroup("backup"), BackupSchedule: viper.GetString(backupCronScheduleFlg), @@ -157,9 +177,10 @@ var startCmd = &cobra.Command{ BackupProvider: bp, Metrics: metrics, Compressor: comp, + Encrypter: encrypter, }) - if err := initializer.New(logger.WithGroup("initializer"), addr, db, bp, comp, metrics, viper.GetString(databaseDatadirFlg)).Start(stop, backuper); err != nil { + if err := initializer.New(logger.WithGroup("initializer"), addr, db, bp, comp, metrics, viper.GetString(databaseDatadirFlg), encrypter).Start(stop, backuper); err != nil { return err } @@ -262,6 +283,58 @@ var waitCmd = &cobra.Command{ }, } +var downloadBackupCmd = &cobra.Command{ + Use: "download", + Short: "downloads backup without restoring", + PreRunE: func(cm *cobra.Command, args []string) error { + err := initEncrypter() + if err != nil { + return err + } + return initBackupProvider() + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("no version argument specified") + } + + c, err := client.New(context.Background(), viper.GetString(serverAddrFlg)) + if err != nil { + return fmt.Errorf("error creating client: %w", err) + } + + backup, err := c.BackupServiceClient().GetBackupByVersion(context.Background(), &v1.GetBackupByVersionRequest{Version: args[0]}) + + if err != nil { + return fmt.Errorf("error getting backup by version: %w", err) + } + + copyPath, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting current dir: %w", err) + } + + if len(args) == 2 { + copyPath = args[1] + } + + destination, err := bp.DownloadBackup(context.Background(), &providers.BackupVersion{Name: backup.Backup.Name}, copyPath) + + if err != nil { + return fmt.Errorf("failed downloading backup: %w", err) + } + + if encrypter != nil { + _, err = encrypter.Decrypt(destination) + if err != nil { + return fmt.Errorf("failed to decrypt: %w", err) + } + } + + return nil + }, +} + func main() { if err := rootCmd.Execute(); err != nil { if logger == nil { @@ -273,7 +346,7 @@ func main() { } func init() { - rootCmd.AddCommand(startCmd, waitCmd, restoreCmd, createBackupCmd) + rootCmd.AddCommand(startCmd, waitCmd, restoreCmd, createBackupCmd, downloadBackupCmd) rootCmd.PersistentFlags().StringP(logLevelFlg, "", "info", "sets the application log level") rootCmd.PersistentFlags().StringP(databaseFlg, "", "", "the kind of the database [postgres|rethinkdb|etcd|meilisearch|redis|keydb|localfs]") @@ -322,6 +395,8 @@ func init() { startCmd.Flags().StringP(compressionMethod, "", "targz", "the compression method to use to compress the backups (tar|targz|tarlz4)") + startCmd.Flags().StringP(encryptionKey, "", "01234567891234560123456789123456", "the encryption key for aes") + err = viper.BindPFlags(startCmd.Flags()) if err != nil { fmt.Printf("unable to construct initializer command: %v", err) @@ -476,6 +551,19 @@ func initDatabase() error { return nil } +func initEncrypter() error { + var err error + key := viper.GetString(encryptionKey) + if key != "" { + encrypter, err = encryption.New(logger.WithGroup("encryption"), key) + if err != nil { + return fmt.Errorf("unable to initialize encryption:%v", err) + } + logger.Info("initialized encrypter") + } + return nil +} + func initBackupProvider() error { bpString := viper.GetString(backupProviderFlg) var err error diff --git a/go.sum b/go.sum index b76fe99..50d4e78 100644 --- a/go.sum +++ b/go.sum @@ -306,8 +306,6 @@ github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/j github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= -github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= diff --git a/proto/v1/backup.proto b/proto/v1/backup.proto index 1eb0590..c1e9f99 100644 --- a/proto/v1/backup.proto +++ b/proto/v1/backup.proto @@ -7,6 +7,7 @@ import "google/protobuf/timestamp.proto"; service BackupService { rpc ListBackups(ListBackupsRequest) returns (BackupListResponse); rpc RestoreBackup(RestoreBackupRequest) returns (RestoreBackupResponse); + rpc GetBackupByVersion(GetBackupByVersionRequest) returns (GetBackupByVersionResponse); } message ListBackupsRequest {} @@ -26,3 +27,11 @@ message RestoreBackupRequest { } message RestoreBackupResponse {} + +message GetBackupByVersionRequest { + string version = 1; +} + +message GetBackupByVersionResponse { + Backup backup = 1; +} From 4ceeefcc2534b1bfb318054cef8b12000bddd261 Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 7 Oct 2024 10:27:39 +0200 Subject: [PATCH 02/16] added and adjusted tests --- .../backup/providers/common/common_test.go | 24 ++++++++ .../providers/gcp/gcp_integration_test.go | 9 +-- .../backup/providers/local/local_test.go | 7 +-- .../providers/s3/s3_integration_test.go | 9 +-- cmd/internal/encryption/encryption_test.go | 56 +++++++++++++++++++ cmd/internal/initializer/initializer.go | 9 +-- cmd/internal/initializer/service.go | 7 +-- cmd/main.go | 1 - 8 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 cmd/internal/encryption/encryption_test.go diff --git a/cmd/internal/backup/providers/common/common_test.go b/cmd/internal/backup/providers/common/common_test.go index 5d161cf..af881cf 100644 --- a/cmd/internal/backup/providers/common/common_test.go +++ b/cmd/internal/backup/providers/common/common_test.go @@ -69,3 +69,27 @@ func TestLatest(t *testing.T) { }) } } + +func TestDetermineBackupFilePath(t *testing.T) { + outPath1 := "" + outPath2 := "/backup/test" + downloadDir := "/backup/restore" + fileName := "0.tar.aes" + + wantBackupFilePath1 := "/backup/restore/0.tar.aes" + wantBackupFilePath2 := "/backup/test/0.tar.aes" + + t.Run("empty-outpath", func(t *testing.T) { + backupFilePath := DeterminBackupFilePath(outPath1, downloadDir, fileName) + if backupFilePath != wantBackupFilePath1 { + t.Errorf("DetermineBackupFilePath() = %v, want %v", backupFilePath, wantBackupFilePath1) + } + }) + + t.Run("filled-outpath", func(t *testing.T) { + backupFilePath := DeterminBackupFilePath(outPath2, downloadDir, fileName) + if backupFilePath != wantBackupFilePath2 { + t.Errorf("DetermineBackupFilePath() = %v, want %v", backupFilePath, wantBackupFilePath2) + } + }) +} diff --git a/cmd/internal/backup/providers/gcp/gcp_integration_test.go b/cmd/internal/backup/providers/gcp/gcp_integration_test.go index ec87167..bba0880 100644 --- a/cmd/internal/backup/providers/gcp/gcp_integration_test.go +++ b/cmd/internal/backup/providers/gcp/gcp_integration_test.go @@ -1,5 +1,3 @@ -//go:build integration - package gcp import ( @@ -153,18 +151,17 @@ func Test_BackupProviderGCP(t *testing.T) { latestVersion := versions.Latest() require.NotNil(t, latestVersion) - err = p.DownloadBackup(ctx, latestVersion) + backupFilePath, err := p.DownloadBackup(ctx, latestVersion, "") require.NoError(t, err) - downloadPath := path.Join(constants.DownloadDir, expectedBackupName) - gotContent, err := afero.ReadFile(fs, downloadPath) + gotContent, err := afero.ReadFile(fs, backupFilePath) require.NoError(t, err) backupContent := fmt.Sprintf("precious data %d", backupAmount-1) require.Equal(t, backupContent, string(gotContent)) // cleaning up after test - err = fs.Remove(downloadPath) + err = fs.Remove(backupFilePath) require.NoError(t, err) }) diff --git a/cmd/internal/backup/providers/local/local_test.go b/cmd/internal/backup/providers/local/local_test.go index 44d528d..05286d6 100644 --- a/cmd/internal/backup/providers/local/local_test.go +++ b/cmd/internal/backup/providers/local/local_test.go @@ -130,17 +130,16 @@ func Test_BackupProviderLocal(t *testing.T) { latestVersion := versions.Latest() require.NotNil(t, latestVersion) - err = p.DownloadBackup(ctx, latestVersion) + backupFilePath, err := p.DownloadBackup(ctx, latestVersion, "") require.NoError(t, err) - downloadPath := path.Join(constants.DownloadDir, latestVersion.Name) - gotContent, err := afero.ReadFile(fs, downloadPath) + gotContent, err := afero.ReadFile(fs, backupFilePath) require.NoError(t, err) require.Equal(t, fmt.Sprintf("precious data %d", backupAmount), string(gotContent)) // cleaning up after test - err = fs.Remove(downloadPath) + err = fs.Remove(backupFilePath) require.NoError(t, err) }) diff --git a/cmd/internal/backup/providers/s3/s3_integration_test.go b/cmd/internal/backup/providers/s3/s3_integration_test.go index 4efcbaa..eab66e5 100644 --- a/cmd/internal/backup/providers/s3/s3_integration_test.go +++ b/cmd/internal/backup/providers/s3/s3_integration_test.go @@ -1,5 +1,3 @@ -//go:build integration - package s3 import ( @@ -147,18 +145,17 @@ func Test_BackupProviderS3(t *testing.T) { latestVersion := versions.Latest() require.NotNil(t, latestVersion) - err = p.DownloadBackup(ctx, latestVersion) + backupFilePath, err := p.DownloadBackup(ctx, latestVersion, "") require.NoError(t, err) - downloadPath := path.Join(constants.DownloadDir, expectedBackupName) - gotContent, err := afero.ReadFile(fs, downloadPath) + gotContent, err := afero.ReadFile(fs, backupFilePath) require.NoError(t, err) backupContent := fmt.Sprintf("precious data %d", backupAmount-1) require.Equal(t, backupContent, string(gotContent)) // cleaning up after test - err = fs.Remove(downloadPath) + err = fs.Remove(backupFilePath) require.NoError(t, err) }) diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go new file mode 100644 index 0000000..10c03e9 --- /dev/null +++ b/cmd/internal/encryption/encryption_test.go @@ -0,0 +1,56 @@ +package encryption + +import ( + "log/slog" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEncrypter(t *testing.T) { + // Key too short + _, err := New(slog.Default(), "tooshortkey") + assert.EqualError(t, err, "key length:11 invalid, must be 16,24 or 32 bytes") + + // Key too short + _, err = New(slog.Default(), "19bytesofencryption") + assert.EqualError(t, err, "key length:19 invalid, must be 16,24 or 32 bytes") + + // Key too long + _, err = New(slog.Default(), "tooloooonoooooooooooooooooooooooooooongkey") + assert.EqualError(t, err, "key length:42 invalid, must be 16,24 or 32 bytes") + + e, err := New(slog.Default(), "01234567891234560123456789123456") + assert.NoError(t, err, "") + + input, err := os.CreateTemp("", "encrypt") + assert.NoError(t, err) + defer os.Remove(input.Name()) + + cleartextInput := []byte("This is the content of the file") + err = os.WriteFile(input.Name(), cleartextInput, 0644) + assert.NoError(t, err) + output, err := e.Encrypt(input.Name()) + assert.NoError(t, err) + + assert.Equal(t, input.Name()+Suffix, output) + + cleartextFile, err := e.Decrypt(output) + assert.NoError(t, err) + cleartext, err := os.ReadFile(cleartextFile) + assert.NoError(t, err) + assert.Equal(t, cleartextInput, cleartext) + + // Test with 100MB file + bigBuff := make([]byte, 100000000) + err = os.WriteFile("bigfile.test", bigBuff, 0666) + assert.NoError(t, err) + + bigEncFile, err := e.Encrypt("bigfile.test") + assert.NoError(t, err) + _, err = e.Decrypt(bigEncFile) + assert.NoError(t, err) + os.Remove("bigfile.test") + os.Remove("bigfile.test.aes") +} diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index 08ca672..9bd54bc 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -166,7 +166,7 @@ func (i *Initializer) initialize(ctx context.Context) error { return nil } - err = i.Restore(ctx, latestBackup, false) + err = i.Restore(ctx, latestBackup) if err != nil { return fmt.Errorf("unable to restore database: %w", err) } @@ -175,7 +175,7 @@ func (i *Initializer) initialize(ctx context.Context) error { } // Restore restores the database with the given backup version -func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVersion, downloadOnly bool) error { +func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVersion) error { i.log.Info("restoring backup", "version", version.Version, "date", version.Date.String()) i.currentStatus.Status = v1.StatusResponse_RESTORING @@ -218,11 +218,6 @@ func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVers return fmt.Errorf("unable to uncompress backup: %w", err) } - if downloadOnly { - i.log.Info("downloadOnly was specified, skipping database recovery") - return nil - } - i.currentStatus.Message = "restoring backup" err = i.db.Recover(ctx) if err != nil { diff --git a/cmd/internal/initializer/service.go b/cmd/internal/initializer/service.go index a0146a1..fdc70ad 100644 --- a/cmd/internal/initializer/service.go +++ b/cmd/internal/initializer/service.go @@ -28,10 +28,10 @@ func (s *initializerService) Status(context.Context, *v1.StatusRequest) (*v1.Sta type backupService struct { bp providers.BackupProvider - restoreFn func(ctx context.Context, version *providers.BackupVersion, downloadOnly bool) error + restoreFn func(ctx context.Context, version *providers.BackupVersion) error } -func newBackupProviderService(bp providers.BackupProvider, restoreFn func(ctx context.Context, version *providers.BackupVersion, downloadOnly bool) error) *backupService { +func newBackupProviderService(bp providers.BackupProvider, restoreFn func(ctx context.Context, version *providers.BackupVersion) error) *backupService { return &backupService{ bp: bp, restoreFn: restoreFn, @@ -76,8 +76,7 @@ func (s *backupService) RestoreBackup(ctx context.Context, req *v1.RestoreBackup return nil, status.Error(codes.Internal, err.Error()) } - //TODO -> check false value here - err = s.restoreFn(ctx, version, false) + err = s.restoreFn(ctx, version) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("error restoring backup: %s", err)) } diff --git a/cmd/main.go b/cmd/main.go index ff27bed..75a0f32 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -50,7 +50,6 @@ const ( databaseFlg = "db" databaseDatadirFlg = "db-data-directory" - downloadOnlyFlg = "download-only" preExecCommandsFlg = "pre-exec-cmds" postExecCommandsFlg = "post-exec-cmds" From 8aec1483adf6fd480ff34ecd77fc03d8d90889e6 Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 7 Oct 2024 10:34:47 +0200 Subject: [PATCH 03/16] adjusted documentation with encryption --- README.md | 6 ++++++ docs/sequence.drawio.svg | 44 +++++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 44f450a..d377287 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ With `--compression-method` you can define how generated backups are compressed - S3 Buckets (tested against Ceph RADOS gateway) - Local +## Encryption + +For all three storage providers are AES encryption is supported, can be enabled with `--encryption-key=`. +The key must be either 16 bytes (AES-128), 24 bytes (AES-192) or 32 bytes (AES-256) long. +The backups are stored at the storage provider with the `.aes` suffix. If the file does not have this suffix, decryption is skipped. + ## How it works ![Sequence Diagram](docs/sequence.drawio.svg) diff --git a/docs/sequence.drawio.svg b/docs/sequence.drawio.svg index 6c39927..94003db 100644 --- a/docs/sequence.drawio.svg +++ b/docs/sequence.drawio.svg @@ -1,4 +1,4 @@ - + @@ -73,18 +73,18 @@ -
+
- uncompress backup archive + (decrypt), uncompress backup archive
and restore database
- - uncompress backup archive... + + (decrypt), uncompress backup archive... @@ -226,7 +226,7 @@ -
+
backup version @@ -234,7 +234,7 @@
- + backup version @@ -244,7 +244,7 @@ -
+
download backup version @@ -252,17 +252,17 @@
- + download backup version - - + + -
+
backup archive @@ -270,7 +270,7 @@
- + backup archive @@ -327,7 +327,7 @@ -
+
probe @@ -335,7 +335,7 @@
- + probe @@ -381,17 +381,19 @@ -
+
compress to
- backup archive + backup archive, +
+ (encrypt)
- + compress to... @@ -474,7 +476,7 @@ -
+
status checking @@ -482,7 +484,7 @@
- + status checking @@ -654,4 +656,4 @@ - + \ No newline at end of file From 3fad9c9d55d7d2348d94b71f034a05f2c1951451 Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 7 Oct 2024 11:21:40 +0200 Subject: [PATCH 04/16] fix linting and tests --- README.md | 2 +- .../providers/gcp/gcp_integration_test.go | 2 ++ .../providers/s3/s3_integration_test.go | 2 ++ cmd/internal/encryption/encryption.go | 5 +-- cmd/internal/encryption/encryption_test.go | 34 +++++++++---------- cmd/main.go | 6 ++-- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d377287..3c17c20 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ With `--compression-method` you can define how generated backups are compressed ## Encryption -For all three storage providers are AES encryption is supported, can be enabled with `--encryption-key=`. +For all three storage providers AES encryption is supported and can be enabled with `--encryption-key=`. The key must be either 16 bytes (AES-128), 24 bytes (AES-192) or 32 bytes (AES-256) long. The backups are stored at the storage provider with the `.aes` suffix. If the file does not have this suffix, decryption is skipped. diff --git a/cmd/internal/backup/providers/gcp/gcp_integration_test.go b/cmd/internal/backup/providers/gcp/gcp_integration_test.go index bba0880..caf22dc 100644 --- a/cmd/internal/backup/providers/gcp/gcp_integration_test.go +++ b/cmd/internal/backup/providers/gcp/gcp_integration_test.go @@ -1,3 +1,5 @@ +//go:build integration + package gcp import ( diff --git a/cmd/internal/backup/providers/s3/s3_integration_test.go b/cmd/internal/backup/providers/s3/s3_integration_test.go index eab66e5..20c3e03 100644 --- a/cmd/internal/backup/providers/s3/s3_integration_test.go +++ b/cmd/internal/backup/providers/s3/s3_integration_test.go @@ -1,3 +1,5 @@ +//go:build integration + package s3 import ( diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index 4665716..eb6c799 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -4,6 +4,7 @@ import ( "crypto/aes" "crypto/cipher" "crypto/rand" + "errors" "fmt" "io" "log/slog" @@ -85,7 +86,7 @@ func (e *Encrypter) Encrypt(input string) (string, error) { } } - if err == io.EOF { + if errors.Is(err, io.EOF) { break } @@ -165,7 +166,7 @@ func (e *Encrypter) Decrypt(input string) (string, error) { } } - if err == io.EOF { + if errors.Is(err, io.EOF) { break } diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go index 10c03e9..2e23ae7 100644 --- a/cmd/internal/encryption/encryption_test.go +++ b/cmd/internal/encryption/encryption_test.go @@ -5,52 +5,52 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestEncrypter(t *testing.T) { // Key too short _, err := New(slog.Default(), "tooshortkey") - assert.EqualError(t, err, "key length:11 invalid, must be 16,24 or 32 bytes") + require.EqualError(t, err, "key length:11 invalid, must be 16,24 or 32 bytes") // Key too short _, err = New(slog.Default(), "19bytesofencryption") - assert.EqualError(t, err, "key length:19 invalid, must be 16,24 or 32 bytes") + require.EqualError(t, err, "key length:19 invalid, must be 16,24 or 32 bytes") // Key too long _, err = New(slog.Default(), "tooloooonoooooooooooooooooooooooooooongkey") - assert.EqualError(t, err, "key length:42 invalid, must be 16,24 or 32 bytes") + require.EqualError(t, err, "key length:42 invalid, must be 16,24 or 32 bytes") e, err := New(slog.Default(), "01234567891234560123456789123456") - assert.NoError(t, err, "") + require.NoError(t, err, "") input, err := os.CreateTemp("", "encrypt") - assert.NoError(t, err) + require.NoError(t, err) defer os.Remove(input.Name()) cleartextInput := []byte("This is the content of the file") - err = os.WriteFile(input.Name(), cleartextInput, 0644) - assert.NoError(t, err) + err = os.WriteFile(input.Name(), cleartextInput, 0600) + require.NoError(t, err) output, err := e.Encrypt(input.Name()) - assert.NoError(t, err) + require.NoError(t, err) - assert.Equal(t, input.Name()+Suffix, output) + require.Equal(t, input.Name()+Suffix, output) cleartextFile, err := e.Decrypt(output) - assert.NoError(t, err) + require.NoError(t, err) cleartext, err := os.ReadFile(cleartextFile) - assert.NoError(t, err) - assert.Equal(t, cleartextInput, cleartext) + require.NoError(t, err) + require.Equal(t, cleartextInput, cleartext) // Test with 100MB file bigBuff := make([]byte, 100000000) - err = os.WriteFile("bigfile.test", bigBuff, 0666) - assert.NoError(t, err) + err = os.WriteFile("bigfile.test", bigBuff, 0600) + require.NoError(t, err) bigEncFile, err := e.Encrypt("bigfile.test") - assert.NoError(t, err) + require.NoError(t, err) _, err = e.Decrypt(bigEncFile) - assert.NoError(t, err) + require.NoError(t, err) os.Remove("bigfile.test") os.Remove("bigfile.test.aes") } diff --git a/cmd/main.go b/cmd/main.go index 75a0f32..dcf96ac 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -162,7 +162,7 @@ var startCmd = &cobra.Command{ if key != "" { encrypter, err = encryption.New(logger.WithGroup("encryption"), key) if err != nil { - return fmt.Errorf("unable to initialize encryption:%v", err) + return fmt.Errorf("unable to initialize encryption: %w", err) } logger.Info("successfully initialized encrypter") } else { @@ -317,7 +317,7 @@ var downloadBackupCmd = &cobra.Command{ copyPath = args[1] } - destination, err := bp.DownloadBackup(context.Background(), &providers.BackupVersion{Name: backup.Backup.Name}, copyPath) + destination, err := bp.DownloadBackup(context.Background(), &providers.BackupVersion{Name: backup.GetBackup().GetName()}, copyPath) if err != nil { return fmt.Errorf("failed downloading backup: %w", err) @@ -556,7 +556,7 @@ func initEncrypter() error { if key != "" { encrypter, err = encryption.New(logger.WithGroup("encryption"), key) if err != nil { - return fmt.Errorf("unable to initialize encryption:%v", err) + return fmt.Errorf("unable to initialize encryption: %w", err) } logger.Info("initialized encrypter") } From 5fb9a182e08f4d061e0f456548330d31c6c5a0e7 Mon Sep 17 00:00:00 2001 From: ostempel Date: Tue, 8 Oct 2024 11:39:03 +0200 Subject: [PATCH 05/16] remove DetermineBackupFilePath and provide outDir directly --- .../backup/providers/common/common.go | 12 ---------- .../backup/providers/common/common_test.go | 24 ------------------- cmd/internal/backup/providers/gcp/gcp.go | 7 +++--- cmd/internal/backup/providers/local/local.go | 7 +++--- cmd/internal/backup/providers/s3/s3.go | 7 +++--- cmd/internal/encryption/encryption.go | 10 ++++---- cmd/internal/encryption/encryption_test.go | 2 +- cmd/internal/initializer/initializer.go | 2 +- 8 files changed, 16 insertions(+), 55 deletions(-) diff --git a/cmd/internal/backup/providers/common/common.go b/cmd/internal/backup/providers/common/common.go index a09cfcb..11917ab 100644 --- a/cmd/internal/backup/providers/common/common.go +++ b/cmd/internal/backup/providers/common/common.go @@ -2,7 +2,6 @@ package common import ( "fmt" - "path/filepath" "sort" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" @@ -33,14 +32,3 @@ func Get(versions []*providers.BackupVersion, version string) (*providers.Backup } return nil, fmt.Errorf("version %q not found", version) } - -// Determines whether a filepath is specified or not (uses default downloadDir) -func DeterminBackupFilePath(outPath string, downloadDir string, fileName string) string { - backupFilePath := "" - if outPath == "" { - backupFilePath = filepath.Join(downloadDir, fileName) - } else { - backupFilePath = filepath.Join(outPath, fileName) - } - return backupFilePath -} diff --git a/cmd/internal/backup/providers/common/common_test.go b/cmd/internal/backup/providers/common/common_test.go index af881cf..5d161cf 100644 --- a/cmd/internal/backup/providers/common/common_test.go +++ b/cmd/internal/backup/providers/common/common_test.go @@ -69,27 +69,3 @@ func TestLatest(t *testing.T) { }) } } - -func TestDetermineBackupFilePath(t *testing.T) { - outPath1 := "" - outPath2 := "/backup/test" - downloadDir := "/backup/restore" - fileName := "0.tar.aes" - - wantBackupFilePath1 := "/backup/restore/0.tar.aes" - wantBackupFilePath2 := "/backup/test/0.tar.aes" - - t.Run("empty-outpath", func(t *testing.T) { - backupFilePath := DeterminBackupFilePath(outPath1, downloadDir, fileName) - if backupFilePath != wantBackupFilePath1 { - t.Errorf("DetermineBackupFilePath() = %v, want %v", backupFilePath, wantBackupFilePath1) - } - }) - - t.Run("filled-outpath", func(t *testing.T) { - backupFilePath := DeterminBackupFilePath(outPath2, downloadDir, fileName) - if backupFilePath != wantBackupFilePath2 { - t.Errorf("DetermineBackupFilePath() = %v, want %v", backupFilePath, wantBackupFilePath2) - } - }) -} diff --git a/cmd/internal/backup/providers/gcp/gcp.go b/cmd/internal/backup/providers/gcp/gcp.go index 2e1e988..cd3e51b 100644 --- a/cmd/internal/backup/providers/gcp/gcp.go +++ b/cmd/internal/backup/providers/gcp/gcp.go @@ -13,7 +13,6 @@ import ( "errors" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" - "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "github.com/spf13/afero" @@ -147,8 +146,8 @@ func (b *BackupProviderGCP) CleanupBackups(_ context.Context) error { return nil } -// DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outPath string) (string, error) { +// DownloadBackup downloads the given backup version to the specified folder +func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outDir string) (string, error) { gen, err := strconv.ParseInt(version.Version, 10, 64) if err != nil { return "", err @@ -161,7 +160,7 @@ func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *provide downloadFileName = filepath.Base(downloadFileName) } - backupFilePath := common.DeterminBackupFilePath(outPath, constants.DownloadDir, downloadFileName) + backupFilePath := filepath.Join(outDir, downloadFileName) b.log.Info("downloading", "object", version.Name, "gen", gen, "to", backupFilePath) diff --git a/cmd/internal/backup/providers/local/local.go b/cmd/internal/backup/providers/local/local.go index 26d1e56..8174003 100644 --- a/cmd/internal/backup/providers/local/local.go +++ b/cmd/internal/backup/providers/local/local.go @@ -11,7 +11,6 @@ import ( "errors" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" - "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "github.com/spf13/afero" @@ -86,13 +85,13 @@ func (b *BackupProviderLocal) CleanupBackups(_ context.Context) error { return nil } -// DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderLocal) DownloadBackup(_ context.Context, version *providers.BackupVersion, outPath string) (string, error) { +// DownloadBackup downloads the given backup version to the specified folder +func (b *BackupProviderLocal) DownloadBackup(_ context.Context, version *providers.BackupVersion, outDir string) (string, error) { b.log.Info("download backup called for provider local") source := filepath.Join(b.config.LocalBackupPath, version.Name) - backupFilePath := common.DeterminBackupFilePath(outPath, constants.DownloadDir, version.Name) + backupFilePath := filepath.Join(outDir, version.Name) err := utils.Copy(b.fs, source, backupFilePath) if err != nil { diff --git a/cmd/internal/backup/providers/s3/s3.go b/cmd/internal/backup/providers/s3/s3.go index 7964014..7634a95 100644 --- a/cmd/internal/backup/providers/s3/s3.go +++ b/cmd/internal/backup/providers/s3/s3.go @@ -9,7 +9,6 @@ import ( "errors" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" - "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "github.com/spf13/afero" @@ -189,8 +188,8 @@ func (b *BackupProviderS3) CleanupBackups(_ context.Context) error { return nil } -// DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outPath string) (string, error) { +// DownloadBackup downloads the given backup version to the specified folder +func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outDir string) (string, error) { bucket := aws.String(b.config.BucketName) downloadFileName := version.Name @@ -198,7 +197,7 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider downloadFileName = filepath.Base(downloadFileName) } - backupFilePath := common.DeterminBackupFilePath(outPath, constants.DownloadDir, downloadFileName) + backupFilePath := filepath.Join(outDir, downloadFileName) f, err := b.fs.Create(backupFilePath) if err != nil { diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index eb6c799..d983f58 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -16,7 +16,7 @@ import ( ) // Suffix is appended on encryption and removed on decryption from given input -const Suffix = ".aes" +const suffix = ".aes" // Encrypter is used to encrypt/decrypt backups type Encrypter struct { @@ -45,7 +45,7 @@ func New(log *slog.Logger, key string) (*Encrypter, error) { } func (e *Encrypter) Encrypt(input string) (string, error) { - output := input + Suffix + output := input + suffix e.log.Info("encrypt", "input", input, "output", output) infile, err := os.Open(input) if err != nil { @@ -109,10 +109,10 @@ func (e *Encrypter) Encrypt(input string) (string, error) { // Decrypt input file with key and store decrypted result with suffix removed // if input does not end with suffix, it is assumed that the file was not encrypted. func (e *Encrypter) Decrypt(input string) (string, error) { - output := strings.TrimSuffix(input, Suffix) - e.log.Info("decrypt", "input", input, "output", output) + output := strings.TrimSuffix(input, suffix) + e.log.Debug("decrypt", "input", input, "output", output) extension := filepath.Ext(input) - if extension != Suffix { + if extension != suffix { return input, fmt.Errorf("input is not encrypted") } infile, err := os.Open(input) diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go index 2e23ae7..f41bb7b 100644 --- a/cmd/internal/encryption/encryption_test.go +++ b/cmd/internal/encryption/encryption_test.go @@ -34,7 +34,7 @@ func TestEncrypter(t *testing.T) { output, err := e.Encrypt(input.Name()) require.NoError(t, err) - require.Equal(t, input.Name()+Suffix, output) + require.Equal(t, input.Name()+suffix, output) cleartextFile, err := e.Decrypt(output) require.NoError(t, err) diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index 9bd54bc..744bc4f 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -200,7 +200,7 @@ func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVers return fmt.Errorf("could not delete priorly downloaded file: %w", err) } - _, err := i.bp.DownloadBackup(ctx, version, "") + backupFilePath, err := i.bp.DownloadBackup(ctx, version, constants.DownloadDir) if err != nil { return fmt.Errorf("unable to download backup: %w", err) } From fb2072ff7679724944407627ec23c4cd42f7d7f5 Mon Sep 17 00:00:00 2001 From: ostempel Date: Tue, 8 Oct 2024 14:04:22 +0200 Subject: [PATCH 06/16] add encryption to integration tests --- deploy/etcd-local.yaml | 1 + deploy/keydb-local.yaml | 1 + deploy/localfs-local.yaml | 1 + deploy/meilisearch-local.yaml | 1 + deploy/postgres-local.yaml | 1 + deploy/redis-local.yaml | 1 + deploy/rethinkdb-local.yaml | 1 + pkg/generate/examples/examples/etcd.go | 1 + pkg/generate/examples/examples/keydb.go | 1 + pkg/generate/examples/examples/localfs.go | 1 + pkg/generate/examples/examples/meilisearch.go | 1 + pkg/generate/examples/examples/postgres.go | 1 + pkg/generate/examples/examples/redis.go | 1 + pkg/generate/examples/examples/rethinkdb.go | 1 + 14 files changed, 14 insertions(+) diff --git a/deploy/etcd-local.yaml b/deploy/etcd-local.yaml index b9de8a6..a1274d5 100644 --- a/deploy/etcd-local.yaml +++ b/deploy/etcd-local.yaml @@ -145,6 +145,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: etcd-test etcd-endpoints: http://localhost:32379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - etcd --data-dir=/data/etcd --listen-client-urls http://0.0.0.0:32379 --advertise-client-urls http://0.0.0.0:32379 --listen-peer-urls http://0.0.0.0:32380 --initial-advertise-peer-urls http://0.0.0.0:32380 --initial-cluster default=http://0.0.0.0:32380 --listen-metrics-urls http://0.0.0.0:32381 kind: ConfigMap diff --git a/deploy/keydb-local.yaml b/deploy/keydb-local.yaml index 8487f2a..d8288dd 100644 --- a/deploy/keydb-local.yaml +++ b/deploy/keydb-local.yaml @@ -140,6 +140,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: keydb-test redis-addr: localhost:6379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - keydb-server kind: ConfigMap diff --git a/deploy/localfs-local.yaml b/deploy/localfs-local.yaml index 103395e..7174f43 100644 --- a/deploy/localfs-local.yaml +++ b/deploy/localfs-local.yaml @@ -116,6 +116,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: localfs-test redis-addr: localhost:6379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - tail -f /etc/hosts kind: ConfigMap diff --git a/deploy/meilisearch-local.yaml b/deploy/meilisearch-local.yaml index 320a519..72c6c74 100644 --- a/deploy/meilisearch-local.yaml +++ b/deploy/meilisearch-local.yaml @@ -157,6 +157,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: meilisearch-test compression-method: targz + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files kind: ConfigMap diff --git a/deploy/postgres-local.yaml b/deploy/postgres-local.yaml index 12c18bb..443d44e 100644 --- a/deploy/postgres-local.yaml +++ b/deploy/postgres-local.yaml @@ -184,6 +184,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: postgres-test compression-method: tar + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - docker-entrypoint.sh postgres kind: ConfigMap diff --git a/deploy/redis-local.yaml b/deploy/redis-local.yaml index 14b0d9b..101f0bb 100644 --- a/deploy/redis-local.yaml +++ b/deploy/redis-local.yaml @@ -140,6 +140,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: redis-test redis-addr: localhost:6379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - redis-server kind: ConfigMap diff --git a/deploy/rethinkdb-local.yaml b/deploy/rethinkdb-local.yaml index ad1f7bb..8d9465f 100644 --- a/deploy/rethinkdb-local.yaml +++ b/deploy/rethinkdb-local.yaml @@ -141,6 +141,7 @@ data: rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt backup-cron-schedule: "*/1 * * * *" object-prefix: rethinkdb-test + encryption-key: "01234567891234560123456789123456" post-exec-cmds: # IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location - rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} diff --git a/pkg/generate/examples/examples/etcd.go b/pkg/generate/examples/examples/etcd.go index d76cc1c..6c28ea4 100644 --- a/pkg/generate/examples/examples/etcd.go +++ b/pkg/generate/examples/examples/etcd.go @@ -246,6 +246,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: etcd-test etcd-endpoints: http://localhost:32379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - etcd --data-dir=/data/etcd --listen-client-urls http://0.0.0.0:32379 --advertise-client-urls http://0.0.0.0:32379 --listen-peer-urls http://0.0.0.0:32380 --initial-advertise-peer-urls http://0.0.0.0:32380 --initial-cluster default=http://0.0.0.0:32380 --listen-metrics-urls http://0.0.0.0:32381 `, diff --git a/pkg/generate/examples/examples/keydb.go b/pkg/generate/examples/examples/keydb.go index 706768c..dff05b2 100644 --- a/pkg/generate/examples/examples/keydb.go +++ b/pkg/generate/examples/examples/keydb.go @@ -238,6 +238,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: keydb-test redis-addr: localhost:6379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - keydb-server `, diff --git a/pkg/generate/examples/examples/localfs.go b/pkg/generate/examples/examples/localfs.go index 90aa1a6..1af850b 100644 --- a/pkg/generate/examples/examples/localfs.go +++ b/pkg/generate/examples/examples/localfs.go @@ -207,6 +207,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: localfs-test redis-addr: localhost:6379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - tail -f /etc/hosts `, diff --git a/pkg/generate/examples/examples/meilisearch.go b/pkg/generate/examples/examples/meilisearch.go index 39db910..ece9749 100644 --- a/pkg/generate/examples/examples/meilisearch.go +++ b/pkg/generate/examples/examples/meilisearch.go @@ -279,6 +279,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: meilisearch-test compression-method: targz +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files `, diff --git a/pkg/generate/examples/examples/postgres.go b/pkg/generate/examples/examples/postgres.go index 4be6afc..96149cb 100644 --- a/pkg/generate/examples/examples/postgres.go +++ b/pkg/generate/examples/examples/postgres.go @@ -309,6 +309,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: postgres-test compression-method: tar +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - docker-entrypoint.sh postgres `, diff --git a/pkg/generate/examples/examples/redis.go b/pkg/generate/examples/examples/redis.go index ff8ec12..e21b32f 100644 --- a/pkg/generate/examples/examples/redis.go +++ b/pkg/generate/examples/examples/redis.go @@ -238,6 +238,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: redis-test redis-addr: localhost:6379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - redis-server `, diff --git a/pkg/generate/examples/examples/rethinkdb.go b/pkg/generate/examples/examples/rethinkdb.go index dd0305d..a5ac7e9 100644 --- a/pkg/generate/examples/examples/rethinkdb.go +++ b/pkg/generate/examples/examples/rethinkdb.go @@ -261,6 +261,7 @@ backup-provider: local rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt backup-cron-schedule: "*/1 * * * *" object-prefix: rethinkdb-test +encryption-key: "01234567891234560123456789123456" post-exec-cmds: # IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location - rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} From a61d7cb25bc2a292c499bf54c5a02ee211e96464 Mon Sep 17 00:00:00 2001 From: ostempel Date: Tue, 8 Oct 2024 14:29:49 +0200 Subject: [PATCH 07/16] fix debug log level --- cmd/internal/encryption/encryption.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index d983f58..f114866 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -46,7 +46,7 @@ func New(log *slog.Logger, key string) (*Encrypter, error) { func (e *Encrypter) Encrypt(input string) (string, error) { output := input + suffix - e.log.Info("encrypt", "input", input, "output", output) + e.log.Debug("encrypt", "input", input, "output", output) infile, err := os.Open(input) if err != nil { return "", err From 800f84a54a784d60b3410c9677cad143b7e1a030 Mon Sep 17 00:00:00 2001 From: ostempel Date: Wed, 9 Oct 2024 13:33:46 +0200 Subject: [PATCH 08/16] refactor encryption to helper functions and adjust tests --- cmd/internal/encryption/encryption.go | 221 +++++++++++++-------- cmd/internal/encryption/encryption_test.go | 36 ++-- cmd/internal/initializer/initializer.go | 6 + cmd/main.go | 44 ++-- 4 files changed, 179 insertions(+), 128 deletions(-) diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index f114866..30334db 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" "unicode" + + "github.com/spf13/afero" ) // Suffix is appended on encryption and removed on decryption from given input @@ -20,173 +22,222 @@ const suffix = ".aes" // Encrypter is used to encrypt/decrypt backups type Encrypter struct { + fs afero.Fs key string log *slog.Logger } +type EncrypterConfig struct { + FS afero.Fs + Key string +} + // New creates a new Encrypter with the given key. -// The key should be 16 bytes (AES-128), 24 bytes (AES-192) or -// 32 bytes (AES-256) -func New(log *slog.Logger, key string) (*Encrypter, error) { - switch len(key) { - case 16, 24, 32: - default: - return nil, fmt.Errorf("key length:%d invalid, must be 16,24 or 32 bytes", len(key)) - } - if !isASCII(key) { +// The key should be 32 bytes (AES-256) +func New(log *slog.Logger, config *EncrypterConfig) (*Encrypter, error) { + if len(config.Key) != 32 { + return nil, fmt.Errorf("key length: %d invalid, must be 32 bytes", len(config.Key)) + } + if !isASCII(config.Key) { return nil, fmt.Errorf("key must only contain ascii characters") } + if config.FS == nil { + config.FS = afero.NewOsFs() + } return &Encrypter{ - key: key, log: log, + key: config.Key, + fs: config.FS, }, nil } -func (e *Encrypter) Encrypt(input string) (string, error) { - output := input + suffix - e.log.Debug("encrypt", "input", input, "output", output) - infile, err := os.Open(input) +// Encrypt input file with key and store encrypted result with suffix +func (e *Encrypter) Encrypt(inputPath string) (string, error) { + output := inputPath + suffix + e.log.Debug("encrypt", "input", inputPath, "output", output) + infile, err := e.fs.Open(inputPath) if err != nil { return "", err } defer infile.Close() - key := []byte(e.key) - block, err := aes.NewCipher(key) + block, err := e.createCipher() if err != nil { return "", err } - // Never use more than 2^32 random nonces with a given key - // because of the risk of repeat. - iv := make([]byte, block.BlockSize()) - if _, err := io.ReadFull(rand.Reader, iv); err != nil { + iv, err := e.generateIV(block) + if err != nil { return "", err } - outfile, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0777) + outfile, err := e.openOutputFile(output) if err != nil { return "", err } defer outfile.Close() - // The buffer size must be multiple of 16 bytes + if err := e.encryptFile(infile, outfile, block, iv); err != nil { + return "", err + } + + if err := e.fs.Remove(inputPath); err != nil { + e.log.Warn("unable to remove input", "error", err) + } + + return output, nil +} + +// Decrypt input file with key and store decrypted result with suffix removed +// if input does not end with suffix, it is assumed that the file was not encrypted. +func (e *Encrypter) Decrypt(inputPath string) (string, error) { + output := strings.TrimSuffix(inputPath, suffix) + e.log.Debug("decrypt", "input", inputPath, "output", output) + + if err := e.validateInput(inputPath); err != nil { + return "", err + } + + infile, err := e.fs.Open(inputPath) + if err != nil { + return "", err + } + defer infile.Close() + + block, err := e.createCipher() + if err != nil { + return "", err + } + + iv, msgLen, err := e.readIVAndMessageLength(infile, block) + if err != nil { + return "", err + } + + outfile, err := e.openOutputFile(output) + if err != nil { + return "", err + } + + if err := e.decryptFile(infile, outfile, block, iv, msgLen); err != nil { + return "", err + } + + if err := e.fs.Remove(inputPath); err != nil { + e.log.Warn("unable to remove input", "error", err) + } + return output, nil +} + +func isASCII(s string) bool { + for _, c := range s { + if c > unicode.MaxASCII { + return false + } + } + return true +} + +// createCipher() returns new cipher block for encryption/decryption based on encryption-key +func (e *Encrypter) createCipher() (cipher.Block, error) { + key := []byte(e.key) + return aes.NewCipher(key) +} + +func (e *Encrypter) openOutputFile(output string) (afero.File, error) { + return e.fs.OpenFile(output, os.O_RDWR|os.O_CREATE, 0644) +} + +// generateIV() returns unique initalization vector of same size as cipher block for encryption +func (e *Encrypter) generateIV(block cipher.Block) ([]byte, error) { + iv := make([]byte, block.BlockSize()) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return nil, err + } + return iv, nil +} + +// encryptFile() encrypts infile to outfile using CTR mode (cipher and iv) and appends iv for decryption +func (e *Encrypter) encryptFile(infile, outfile afero.File, block cipher.Block, iv []byte) error { buf := make([]byte, 1024) stream := cipher.NewCTR(block, iv) + for { n, err := infile.Read(buf) if n > 0 { stream.XORKeyStream(buf, buf[:n]) - // Write into file - _, err = outfile.Write(buf[:n]) - if err != nil { - return "", err + if _, err := outfile.Write(buf[:n]); err != nil { + return err } } if errors.Is(err, io.EOF) { break } - if err != nil { e.log.Info("Read %d bytes: %v", strconv.Itoa(n), err) break } } - // Append the IV - _, err = outfile.Write(iv) - if err == nil { - err := os.Remove(input) - if err != nil { - e.log.Warn("unable to remove input", "error", err) - } + + if _, err := outfile.Write(iv); err != nil { + return err } - return output, err + return nil } -// Decrypt input file with key and store decrypted result with suffix removed -// if input does not end with suffix, it is assumed that the file was not encrypted. -func (e *Encrypter) Decrypt(input string) (string, error) { - output := strings.TrimSuffix(input, suffix) - e.log.Debug("decrypt", "input", input, "output", output) - extension := filepath.Ext(input) - if extension != suffix { - return input, fmt.Errorf("input is not encrypted") - } - infile, err := os.Open(input) - if err != nil { - return "", err - } - defer infile.Close() - - key := []byte(e.key) - block, err := aes.NewCipher(key) - if err != nil { - return "", err +// validateInput() throws error if input file doesn't have encryption suffix +func (e *Encrypter) validateInput(input string) error { + if filepath.Ext(input) != suffix { + return fmt.Errorf("input is not encrypted") } + return nil +} - // Never use more than 2^32 random nonces with a given key - // because of the risk of repeat. +// readIVAndMessageLength() returns initialization vector and message length for decryption +func (e *Encrypter) readIVAndMessageLength(infile afero.File, block cipher.Block) ([]byte, int64, error) { fi, err := infile.Stat() if err != nil { - return "", err + return nil, 0, err } iv := make([]byte, block.BlockSize()) msgLen := fi.Size() - int64(len(iv)) - _, err = infile.ReadAt(iv, msgLen) - if err != nil { - return "", err + if _, err := infile.ReadAt(iv, msgLen); err != nil { + return nil, 0, err } - outfile, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0777) - if err != nil { - return "", err - } - defer outfile.Close() + return iv, msgLen, nil +} - // The buffer size must be multiple of 16 bytes +// decryptFile() decrypts infile to outfile using CTR mode (cipher and iv) +func (e *Encrypter) decryptFile(infile, outfile afero.File, block cipher.Block, iv []byte, msgLen int64) error { buf := make([]byte, 1024) stream := cipher.NewCTR(block, iv) + for { n, err := infile.Read(buf) if n > 0 { - // The last bytes are the IV, don't belong the original message if n > int(msgLen) { n = int(msgLen) } msgLen -= int64(n) stream.XORKeyStream(buf, buf[:n]) - // Write into file - _, err = outfile.Write(buf[:n]) - if err != nil { - return "", err + if _, err := outfile.Write(buf[:n]); err != nil { + return err } } if errors.Is(err, io.EOF) { break } - if err != nil { e.log.Info("Read %d bytes: %v", strconv.Itoa(n), err) break } } - err = os.Remove(input) - if err != nil { - e.log.Warn("unable to remove input", "error", err) - } - return output, nil -} -func isASCII(s string) bool { - for _, c := range s { - if c > unicode.MaxASCII { - return false - } - } - return true + return nil } diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go index f41bb7b..db2fd80 100644 --- a/cmd/internal/encryption/encryption_test.go +++ b/cmd/internal/encryption/encryption_test.go @@ -2,55 +2,59 @@ package encryption import ( "log/slog" - "os" "testing" + "github.com/spf13/afero" "github.com/stretchr/testify/require" ) func TestEncrypter(t *testing.T) { - // Key too short - _, err := New(slog.Default(), "tooshortkey") - require.EqualError(t, err, "key length:11 invalid, must be 16,24 or 32 bytes") + fs := afero.NewMemMapFs() // Key too short - _, err = New(slog.Default(), "19bytesofencryption") - require.EqualError(t, err, "key length:19 invalid, must be 16,24 or 32 bytes") + _, err := New(slog.Default(), &EncrypterConfig{Key: "tooshortkey", FS: fs}) + require.EqualError(t, err, "key length: 11 invalid, must be 32 bytes") // Key too long - _, err = New(slog.Default(), "tooloooonoooooooooooooooooooooooooooongkey") - require.EqualError(t, err, "key length:42 invalid, must be 16,24 or 32 bytes") + _, err = New(slog.Default(), &EncrypterConfig{Key: "tooloooonoooooooooooooooooooooooooooongkey", FS: fs}) + require.EqualError(t, err, "key length: 42 invalid, must be 32 bytes") + + _, err = New(slog.Default(), &EncrypterConfig{Key: "äöüäöüäöüäöüäöüä", FS: fs}) + require.EqualError(t, err, "key must only contain ascii characters") - e, err := New(slog.Default(), "01234567891234560123456789123456") + e, err := New(slog.Default(), &EncrypterConfig{Key: "01234567891234560123456789123456", FS: fs}) require.NoError(t, err, "") - input, err := os.CreateTemp("", "encrypt") + input, err := fs.Create("encrypt") require.NoError(t, err) - defer os.Remove(input.Name()) + defer fs.Remove(input.Name()) cleartextInput := []byte("This is the content of the file") - err = os.WriteFile(input.Name(), cleartextInput, 0600) + err = afero.WriteFile(fs, input.Name(), cleartextInput, 0600) require.NoError(t, err) output, err := e.Encrypt(input.Name()) require.NoError(t, err) + encryptedText, err := afero.ReadFile(fs, output) + require.NoError(t, err) require.Equal(t, input.Name()+suffix, output) + require.NotEqual(t, cleartextInput, encryptedText) cleartextFile, err := e.Decrypt(output) require.NoError(t, err) - cleartext, err := os.ReadFile(cleartextFile) + cleartext, err := afero.ReadFile(fs, cleartextFile) require.NoError(t, err) require.Equal(t, cleartextInput, cleartext) // Test with 100MB file bigBuff := make([]byte, 100000000) - err = os.WriteFile("bigfile.test", bigBuff, 0600) + err = afero.WriteFile(fs, "bigfile.test", bigBuff, 0600) require.NoError(t, err) bigEncFile, err := e.Encrypt("bigfile.test") require.NoError(t, err) _, err = e.Decrypt(bigEncFile) require.NoError(t, err) - os.Remove("bigfile.test") - os.Remove("bigfile.test.aes") + fs.Remove("bigfile.test") + fs.Remove("bigfile.test.aes") } diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index 744bc4f..af632b8 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -139,6 +139,12 @@ func (i *Initializer) initialize(ctx context.Context) error { return fmt.Errorf("unable to ensure backup bucket: %w", err) } + i.log.Info("ensuring default download directory") + err = os.MkdirAll(constants.DownloadDir, 0755) + if err != nil { + return fmt.Errorf("unable to ensure default download directory: %w", err) + } + i.log.Info("checking database") i.currentStatus.Status = v1.StatusResponse_CHECKING i.currentStatus.Message = "checking database" diff --git a/cmd/main.go b/cmd/main.go index dcf96ac..01992ef 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -95,7 +95,9 @@ const ( compressionMethod = "compression-method" - encryptionKey = "encryption-key" + encryptionKeyFlg = "encryption-key" + + downloadOutputFlg = "output" ) var ( @@ -157,18 +159,6 @@ var startCmd = &cobra.Command{ metrics := metrics.New() metrics.Start(logger.WithGroup("metrics")) - var encrypter *encryption.Encrypter - key := viper.GetString(encryptionKey) - if key != "" { - encrypter, err = encryption.New(logger.WithGroup("encryption"), key) - if err != nil { - return fmt.Errorf("unable to initialize encryption: %w", err) - } - logger.Info("successfully initialized encrypter") - } else { - logger.Info("no encrypter found") - } - backuper := backup.New(&backup.BackuperConfig{ Log: logger.WithGroup("backup"), BackupSchedule: viper.GetString(backupCronScheduleFlg), @@ -283,7 +273,7 @@ var waitCmd = &cobra.Command{ } var downloadBackupCmd = &cobra.Command{ - Use: "download", + Use: "download ", Short: "downloads backup without restoring", PreRunE: func(cm *cobra.Command, args []string) error { err := initEncrypter() @@ -308,16 +298,9 @@ var downloadBackupCmd = &cobra.Command{ return fmt.Errorf("error getting backup by version: %w", err) } - copyPath, err := os.Getwd() - if err != nil { - return fmt.Errorf("error getting current dir: %w", err) - } + output := viper.GetString(downloadOutputFlg) - if len(args) == 2 { - copyPath = args[1] - } - - destination, err := bp.DownloadBackup(context.Background(), &providers.BackupVersion{Name: backup.GetBackup().GetName()}, copyPath) + destination, err := bp.DownloadBackup(context.Background(), &providers.BackupVersion{Name: backup.GetBackup().GetName()}, output) if err != nil { return fmt.Errorf("failed downloading backup: %w", err) @@ -394,7 +377,7 @@ func init() { startCmd.Flags().StringP(compressionMethod, "", "targz", "the compression method to use to compress the backups (tar|targz|tarlz4)") - startCmd.Flags().StringP(encryptionKey, "", "01234567891234560123456789123456", "the encryption key for aes") + startCmd.Flags().StringP(encryptionKeyFlg, "", "", "the encryption key for aes") err = viper.BindPFlags(startCmd.Flags()) if err != nil { @@ -413,6 +396,13 @@ func init() { } restoreCmd.AddCommand(restoreListCmd) + + downloadBackupCmd.Flags().StringP(downloadOutputFlg, "o", constants.DownloadDir, "the target directory for the downloaded backup") + err = viper.BindPFlags(downloadBackupCmd.Flags()) + if err != nil { + fmt.Printf("unable to construct download command: %v", err) + os.Exit(1) + } } func initConfig() { @@ -552,11 +542,11 @@ func initDatabase() error { func initEncrypter() error { var err error - key := viper.GetString(encryptionKey) + key := viper.GetString(encryptionKeyFlg) if key != "" { - encrypter, err = encryption.New(logger.WithGroup("encryption"), key) + encrypter, err = encryption.New(logger.WithGroup("encrypter"), &encryption.EncrypterConfig{Key: key}) if err != nil { - return fmt.Errorf("unable to initialize encryption: %w", err) + return fmt.Errorf("unable to initialize encrypter: %w", err) } logger.Info("initialized encrypter") } From ce9b4a96602c83687aeb379cb51810142409d667 Mon Sep 17 00:00:00 2001 From: ostempel Date: Wed, 9 Oct 2024 13:44:48 +0200 Subject: [PATCH 09/16] fix linting --- cmd/internal/encryption/encryption_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go index db2fd80..fb60aa6 100644 --- a/cmd/internal/encryption/encryption_test.go +++ b/cmd/internal/encryption/encryption_test.go @@ -27,7 +27,6 @@ func TestEncrypter(t *testing.T) { input, err := fs.Create("encrypt") require.NoError(t, err) - defer fs.Remove(input.Name()) cleartextInput := []byte("This is the content of the file") err = afero.WriteFile(fs, input.Name(), cleartextInput, 0600) @@ -55,6 +54,11 @@ func TestEncrypter(t *testing.T) { require.NoError(t, err) _, err = e.Decrypt(bigEncFile) require.NoError(t, err) - fs.Remove("bigfile.test") - fs.Remove("bigfile.test.aes") + + err = fs.Remove(input.Name()) + require.NoError(t, err) + err = fs.Remove("bigfile.test") + require.NoError(t, err) + err = fs.Remove("bigfile.test.aes") + require.NoError(t, err) } From f25990b24d4394b24f2b267cd0ed49874aa48c3f Mon Sep 17 00:00:00 2001 From: ostempel Date: Wed, 9 Oct 2024 13:52:35 +0200 Subject: [PATCH 10/16] fix encryption unit test --- cmd/internal/encryption/encryption_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go index fb60aa6..899ea6f 100644 --- a/cmd/internal/encryption/encryption_test.go +++ b/cmd/internal/encryption/encryption_test.go @@ -59,6 +59,4 @@ func TestEncrypter(t *testing.T) { require.NoError(t, err) err = fs.Remove("bigfile.test") require.NoError(t, err) - err = fs.Remove("bigfile.test.aes") - require.NoError(t, err) } From 7a6cd7922ce43f9effed0d4d7b37f594e03668de Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 14 Oct 2024 10:43:40 +0200 Subject: [PATCH 11/16] adjust readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c17c20..8689124 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ With `--compression-method` you can define how generated backups are compressed ## Encryption For all three storage providers AES encryption is supported and can be enabled with `--encryption-key=`. -The key must be either 16 bytes (AES-128), 24 bytes (AES-192) or 32 bytes (AES-256) long. +The key must be 32 bytes (AES-256) long. The backups are stored at the storage provider with the `.aes` suffix. If the file does not have this suffix, decryption is skipped. ## How it works From 953315178386ad760fda5a1be57c0ed5de36bf06 Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 14 Oct 2024 14:28:37 +0200 Subject: [PATCH 12/16] validate encryption mode change on initalization and download --- cmd/internal/backup/providers/contract.go | 2 +- cmd/internal/encryption/encryption.go | 22 +++++++------- cmd/internal/initializer/initializer.go | 35 ++++++++++++++--------- cmd/main.go | 10 ++++--- integration/main_test.go | 2 ++ 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/cmd/internal/backup/providers/contract.go b/cmd/internal/backup/providers/contract.go index 4740f1c..4356dbd 100644 --- a/cmd/internal/backup/providers/contract.go +++ b/cmd/internal/backup/providers/contract.go @@ -10,7 +10,7 @@ type BackupProvider interface { ListBackups(ctx context.Context) (BackupVersions, error) CleanupBackups(ctx context.Context) error GetNextBackupName(ctx context.Context) string - DownloadBackup(ctx context.Context, version *BackupVersion, outPath string) (string, error) + DownloadBackup(ctx context.Context, version *BackupVersion, outDir string) (string, error) UploadBackup(ctx context.Context, sourcePath string) error } diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index 30334db..213601a 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -17,7 +17,7 @@ import ( "github.com/spf13/afero" ) -// Suffix is appended on encryption and removed on decryption from given input +// suffix is appended on encryption and removed on decryption from given input const suffix = ".aes" // Encrypter is used to encrypt/decrypt backups @@ -96,8 +96,8 @@ func (e *Encrypter) Decrypt(inputPath string) (string, error) { output := strings.TrimSuffix(inputPath, suffix) e.log.Debug("decrypt", "input", inputPath, "output", output) - if err := e.validateInput(inputPath); err != nil { - return "", err + if IsEncrypted(inputPath) { + return "", fmt.Errorf("input is not encrypted") } infile, err := e.fs.Open(inputPath) @@ -177,23 +177,21 @@ func (e *Encrypter) encryptFile(infile, outfile afero.File, block cipher.Block, break } if err != nil { - e.log.Info("Read %d bytes: %v", strconv.Itoa(n), err) + e.log.Info("read %d bytes: %s", strconv.Itoa(n), err) break } } if _, err := outfile.Write(iv); err != nil { - return err + return fmt.Errorf("could not append iv: %w", err) } + return nil } -// validateInput() throws error if input file doesn't have encryption suffix -func (e *Encrypter) validateInput(input string) error { - if filepath.Ext(input) != suffix { - return fmt.Errorf("input is not encrypted") - } - return nil +// IsEncrypted() tests if target file is encrypted +func IsEncrypted(path string) bool { + return filepath.Ext(path) == suffix } // readIVAndMessageLength() returns initialization vector and message length for decryption @@ -234,7 +232,7 @@ func (e *Encrypter) decryptFile(infile, outfile afero.File, block cipher.Block, break } if err != nil { - e.log.Info("Read %d bytes: %v", strconv.Itoa(n), err) + e.log.Info("read %d bytes: %s", strconv.Itoa(n), err) break } } diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index af632b8..a6062c6 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -149,29 +149,35 @@ func (i *Initializer) initialize(ctx context.Context) error { i.currentStatus.Status = v1.StatusResponse_CHECKING i.currentStatus.Message = "checking database" - needsBackup, err := i.db.Check(ctx) + versions, err := i.bp.ListBackups(ctx) if err != nil { - return fmt.Errorf("unable to check data of database: %w", err) + return fmt.Errorf("unable retrieve backup versions: %w", err) } - if !needsBackup { - i.log.Info("database does not need to be restored") + latestBackup := versions.Latest() + if latestBackup == nil { + i.log.Info("there are no backups available, it's a fresh database. allow database to start") return nil } - i.log.Info("database potentially needs to be restored, looking for backup") + if i.encrypter == nil { + if encryption.IsEncrypted(latestBackup.Name) { + return fmt.Errorf("latest backup is encrypted, but no encryption/decryption is configured") + } + } - versions, err := i.bp.ListBackups(ctx) + needsBackup, err := i.db.Check(ctx) if err != nil { - return fmt.Errorf("unable retrieve backup versions: %w", err) + return fmt.Errorf("unable to check data of database: %w", err) } - latestBackup := versions.Latest() - if latestBackup == nil { - i.log.Info("there are no backups available, it's a fresh database. allow database to start") + if !needsBackup { + i.log.Info("database does not need to be restored") return nil } + i.log.Info("database potentially needs to be restored, looking for backup") + err = i.Restore(ctx, latestBackup) if err != nil { return fmt.Errorf("unable to restore database: %w", err) @@ -212,10 +218,13 @@ func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVers } if i.encrypter != nil { - backupFilePath, err = i.encrypter.Decrypt(backupFilePath) - if err != nil { - return fmt.Errorf("unable to decrypt backup: %w", err) + if encryption.IsEncrypted(backupFilePath) { + backupFilePath, err = i.encrypter.Decrypt(backupFilePath) + if err != nil { + return fmt.Errorf("unable to decrypt backup: %w", err) + } } + i.log.Info("restoring unencrypted backup with configured encryption - skipping decryption...") } i.currentStatus.Message = "uncompressing backup" diff --git a/cmd/main.go b/cmd/main.go index 01992ef..90a70f7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -307,12 +307,14 @@ var downloadBackupCmd = &cobra.Command{ } if encrypter != nil { - _, err = encrypter.Decrypt(destination) - if err != nil { - return fmt.Errorf("failed to decrypt: %w", err) + if encryption.IsEncrypted(destination) { + _, err = encrypter.Decrypt(destination) + if err != nil { + return fmt.Errorf("unable to decrypt backup: %w", err) + } } + logger.Info("downloading unencrypted backup with configured encryption - skipping decryption...") } - return nil }, } diff --git a/integration/main_test.go b/integration/main_test.go index c29fec9..75112bb 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -143,6 +143,8 @@ func restoreFlow(t *testing.T, spec *flowSpec) { require.NoError(t, err) require.NotNil(t, backup) + require.True(t, strings.HasSuffix(backup.Name, ".aes")) + t.Log("remove sts and delete data volume") err = c.Delete(ctx, spec.sts(ns.Name)) From 49621ad2eb09c9786602d8d049ee8d53c285c19b Mon Sep 17 00:00:00 2001 From: ostempel Date: Mon, 14 Oct 2024 14:33:24 +0200 Subject: [PATCH 13/16] fix linting --- integration/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/main_test.go b/integration/main_test.go index 75112bb..a9d089e 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -143,7 +143,7 @@ func restoreFlow(t *testing.T, spec *flowSpec) { require.NoError(t, err) require.NotNil(t, backup) - require.True(t, strings.HasSuffix(backup.Name, ".aes")) + require.True(t, strings.HasSuffix(backup.GetName(), ".aes")) t.Log("remove sts and delete data volume") From ab41ca3f1bf7de347d0aff1daaebaa6a46bbf37a Mon Sep 17 00:00:00 2001 From: ostempel Date: Tue, 15 Oct 2024 08:40:49 +0200 Subject: [PATCH 14/16] fix decryption --- cmd/internal/encryption/encryption.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index 213601a..486c7b6 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -96,7 +96,7 @@ func (e *Encrypter) Decrypt(inputPath string) (string, error) { output := strings.TrimSuffix(inputPath, suffix) e.log.Debug("decrypt", "input", inputPath, "output", output) - if IsEncrypted(inputPath) { + if !IsEncrypted(inputPath) { return "", fmt.Errorf("input is not encrypted") } From 5cf14d96099c604eff8382421c5a9a536107f07d Mon Sep 17 00:00:00 2001 From: ostempel Date: Wed, 23 Oct 2024 14:52:49 +0200 Subject: [PATCH 15/16] fix logging message --- cmd/internal/initializer/initializer.go | 3 ++- cmd/main.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index a6062c6..ec43167 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -223,8 +223,9 @@ func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVers if err != nil { return fmt.Errorf("unable to decrypt backup: %w", err) } + } else { + i.log.Info("restoring unencrypted backup with configured encryption - skipping decryption...") } - i.log.Info("restoring unencrypted backup with configured encryption - skipping decryption...") } i.currentStatus.Message = "uncompressing backup" diff --git a/cmd/main.go b/cmd/main.go index 90a70f7..74f9414 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -312,8 +312,9 @@ var downloadBackupCmd = &cobra.Command{ if err != nil { return fmt.Errorf("unable to decrypt backup: %w", err) } + } else { + logger.Info("downloading unencrypted backup with configured encryption - skipping decryption...") } - logger.Info("downloading unencrypted backup with configured encryption - skipping decryption...") } return nil }, From 43b73654c63674ac672bb58d75787c0fc7ce9cb3 Mon Sep 17 00:00:00 2001 From: ostempel Date: Wed, 23 Oct 2024 15:20:58 +0200 Subject: [PATCH 16/16] fix error on file reading and db-check --- cmd/internal/encryption/encryption.go | 7 ++----- cmd/internal/encryption/encryption_test.go | 2 +- cmd/internal/initializer/initializer.go | 24 +++++++++++----------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go index 486c7b6..65b0c85 100644 --- a/cmd/internal/encryption/encryption.go +++ b/cmd/internal/encryption/encryption.go @@ -10,7 +10,6 @@ import ( "log/slog" "os" "path/filepath" - "strconv" "strings" "unicode" @@ -177,8 +176,7 @@ func (e *Encrypter) encryptFile(infile, outfile afero.File, block cipher.Block, break } if err != nil { - e.log.Info("read %d bytes: %s", strconv.Itoa(n), err) - break + return fmt.Errorf("error reading from file (%d bytes read): %w", n, err) } } @@ -232,8 +230,7 @@ func (e *Encrypter) decryptFile(infile, outfile afero.File, block cipher.Block, break } if err != nil { - e.log.Info("read %d bytes: %s", strconv.Itoa(n), err) - break + return fmt.Errorf("error reading from file (%d bytes read): %w", n, err) } } diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go index 899ea6f..bbc3823 100644 --- a/cmd/internal/encryption/encryption_test.go +++ b/cmd/internal/encryption/encryption_test.go @@ -16,7 +16,7 @@ func TestEncrypter(t *testing.T) { require.EqualError(t, err, "key length: 11 invalid, must be 32 bytes") // Key too long - _, err = New(slog.Default(), &EncrypterConfig{Key: "tooloooonoooooooooooooooooooooooooooongkey", FS: fs}) + _, err = New(slog.Default(), &EncrypterConfig{Key: "toolooooooooooooooooooooooooooooooooongkey", FS: fs}) require.EqualError(t, err, "key length: 42 invalid, must be 32 bytes") _, err = New(slog.Default(), &EncrypterConfig{Key: "äöüäöüäöüäöüäöüä", FS: fs}) diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index ec43167..2f11fbb 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -149,6 +149,18 @@ func (i *Initializer) initialize(ctx context.Context) error { i.currentStatus.Status = v1.StatusResponse_CHECKING i.currentStatus.Message = "checking database" + needsBackup, err := i.db.Check(ctx) + if err != nil { + return fmt.Errorf("unable to check data of database: %w", err) + } + + if !needsBackup { + i.log.Info("database does not need to be restored") + return nil + } + + i.log.Info("database potentially needs to be restored, looking for backup") + versions, err := i.bp.ListBackups(ctx) if err != nil { return fmt.Errorf("unable retrieve backup versions: %w", err) @@ -166,18 +178,6 @@ func (i *Initializer) initialize(ctx context.Context) error { } } - needsBackup, err := i.db.Check(ctx) - if err != nil { - return fmt.Errorf("unable to check data of database: %w", err) - } - - if !needsBackup { - i.log.Info("database does not need to be restored") - return nil - } - - i.log.Info("database potentially needs to be restored, looking for backup") - err = i.Restore(ctx, latestBackup) if err != nil { return fmt.Errorf("unable to restore database: %w", err)