diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel index d3c382ce1df78..1b04fdbb88ca5 100644 --- a/br/pkg/streamhelper/BUILD.bazel +++ b/br/pkg/streamhelper/BUILD.bazel @@ -69,7 +69,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 32, + shard_count = 33, deps = [ ":streamhelper", "//br/pkg/errors", diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go index 6d477994ef07f..482b826252d60 100644 --- a/br/pkg/streamhelper/advancer.go +++ b/br/pkg/streamhelper/advancer.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/redact" tikvstore "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" @@ -438,7 +439,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error if err != nil { log.Warn("failed to upload service GC safepoint, skipping.", logutil.ShortError(err)) } - log.Info("added event", zap.Stringer("task", e.Info), + log.Info("added event", zap.Stringer("task", redact.TaskInfoRedacted{Info: e.Info}), zap.Stringer("ranges", logutil.StringifyKeys(c.taskRange)), zap.Uint64("current-checkpoint", p)) case EventDel: utils.LogBackupTaskCountDec() diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go index cab1164dd36e6..90eebd1b8af90 100644 --- a/br/pkg/streamhelper/advancer_test.go +++ b/br/pkg/streamhelper/advancer_test.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/streamhelper/spans" "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/util/redact" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -822,3 +823,53 @@ func TestSubscriptionPanic(t *testing.T) { cancel() wg.Wait() } + +func TestRedactBackend(t *testing.T) { + info := new(backup.StreamBackupTaskInfo) + info.Name = "test" + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_S3{ + S3: &backup.S3{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + AccessKey: "12abCD!@#[]{}?/\\", + SecretAccessKey: "12abCD!@#[]{}?/\\", + }, + }, + } + + redacted := redact.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) + + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_Gcs{ + Gcs: &backup.GCS{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + CredentialsBlob: "12abCD!@#[]{}?/\\", + }, + }, + } + redacted = redact.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) + + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_AzureBlobStorage{ + AzureBlobStorage: &backup.AzureBlobStorage{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + SharedKey: "12abCD!@#[]{}?/\\", + AccessSig: "12abCD!@#[]{}?/\\", + EncryptionKey: &backup.AzureCustomerKey{ + EncryptionKey: "12abCD!@#[]{}?/\\", + EncryptionKeySha256: "12abCD!@#[]{}?/\\", + }, + }, + }, + } + redacted = redact.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > > name:\"test\" ", redacted.String()) +} diff --git a/pkg/util/redact/BUILD.bazel b/pkg/util/redact/BUILD.bazel index 18c8b3c2c2cc7..7349aa43d80c7 100644 --- a/pkg/util/redact/BUILD.bazel +++ b/pkg/util/redact/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/util/intest", "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_kvproto//pkg/brpb", ], ) diff --git a/pkg/util/redact/redact.go b/pkg/util/redact/redact.go index 0ca06fe17160d..5dd0e105ca4dc 100644 --- a/pkg/util/redact/redact.go +++ b/pkg/util/redact/redact.go @@ -22,14 +22,23 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "github.com/pingcap/errors" + backup "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/tidb/pkg/util/intest" ) var ( _ fmt.Stringer = redactStringer{} + + reAccessKey = regexp.MustCompile(`access_key:\"[^\"]*\"`) + reSecretAccessKey = regexp.MustCompile(`secret_access_key:\"[^\"]*\"`) + reSharedKey = regexp.MustCompile(`shared_key:\"[^\"]*\"`) + reCredentialsBlob = regexp.MustCompile(`credentials_blob:\"[^\"]*\"`) + reAccessSig = regexp.MustCompile(`access_sig:\"[^\"]*\"`) + reEncryptKey = regexp.MustCompile(`encryption_key:<.*?>`) ) // String will redact the input string according to 'mode'. Check 'tidb_redact_log': https://github.com/pingcap/tidb/blob/acf9e3128693a5a13f31027f05f4de41edf8d7b2/pkg/sessionctx/variable/sysvar.go#L2154. @@ -223,3 +232,25 @@ func WriteRedact(build *strings.Builder, v string, redact string) { } build.WriteString(v) } + +// TaskInfoRedacted is a wrapper of backup.StreamBackupTaskInfo to redact sensitive information +type TaskInfoRedacted struct { + Info *backup.StreamBackupTaskInfo +} + +func (TaskInfoRedacted) redact(input string) string { + // Replace the matched fields with redacted versions + output := reAccessKey.ReplaceAllString(input, `access_key:"[REDACTED]"`) + output = reSecretAccessKey.ReplaceAllString(output, `secret_access_key:"[REDACTED]"`) + output = reSharedKey.ReplaceAllString(output, `shared_key:"[REDACTED]"`) + output = reCredentialsBlob.ReplaceAllString(output, `CredentialsBlob:"[REDACTED]"`) + output = reAccessSig.ReplaceAllString(output, `access_sig:"[REDACTED]"`) + output = reEncryptKey.ReplaceAllString(output, `encryption_key:<[REDACTED]>`) + + return output +} + +// String returns the redacted string of the task info +func (t TaskInfoRedacted) String() string { + return t.redact(t.Info.String()) +}