Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

placement: supports survival preferences #40613

Merged
merged 27 commits into from
Feb 8, 2023
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
01c7bf4
placement: supports survival preferences
nolouch Feb 6, 2023
46e7d42
Merge remote-tracking branch 'origin/master' into placement-locality
nolouch Feb 6, 2023
e8fcb17
address comments
nolouch Feb 6, 2023
d62f9c7
add tidy logic
nolouch Feb 6, 2023
7c22e7b
fix test
nolouch Feb 6, 2023
e13f86c
Merge branch 'master' into placement-locality
nolouch Feb 6, 2023
3f55670
Merge branch 'master' into placement-locality
nolouch Feb 7, 2023
ac610f9
Merge branch 'master' into placement-locality
nolouch Feb 7, 2023
12bde62
address comments
nolouch Feb 7, 2023
b68dacc
Merge remote-tracking branch 'nolouch/placement-locality' into placem…
nolouch Feb 7, 2023
3e7199d
Merge branch 'master' into placement-locality
nolouch Feb 7, 2023
051e51a
Merge branch 'master' into placement-locality
nolouch Feb 7, 2023
8744005
Merge remote-tracking branch 'origin/master' into placement-locality
nolouch Feb 7, 2023
39e7641
Merge remote-tracking branch 'nolouch/placement-locality' into placem…
nolouch Feb 7, 2023
16640a7
Merge branch 'master' into placement-locality
ti-chi-bot Feb 7, 2023
c2439fa
Merge branch 'master' into placement-locality
ti-chi-bot Feb 7, 2023
62116d2
Merge branch 'master' into placement-locality
ti-chi-bot Feb 7, 2023
9991a42
Merge branch 'master' into placement-locality
ti-chi-bot Feb 7, 2023
bbd8b1c
bazel prepare
nolouch Feb 7, 2023
1f345bc
Merge branch 'master' into placement-locality
ti-chi-bot Feb 7, 2023
bad4f27
Merge branch 'master' into placement-locality
ti-chi-bot Feb 8, 2023
feee147
Merge branch 'master' into placement-locality
ti-chi-bot Feb 8, 2023
ecfac61
Merge branch 'master' into placement-locality
ti-chi-bot Feb 8, 2023
3afb5ac
Merge branch 'master' into placement-locality
ti-chi-bot Feb 8, 2023
c0cebab
Merge remote-tracking branch 'origin' into placement-locality
nolouch Feb 8, 2023
0232740
Merge remote-tracking branch 'nolouch/placement-locality' into placem…
nolouch Feb 8, 2023
838cfd3
Merge branch 'master' into placement-locality
ti-chi-bot Feb 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ddl/ddl_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -3026,6 +3026,8 @@ func SetDirectPlacementOpt(placementSettings *model.PlacementSettings, placement
placementSettings.FollowerConstraints = stringVal
case ast.PlacementOptionVoterConstraints:
placementSettings.VoterConstraints = stringVal
case ast.PlacementOptionSurvivalPreferences:
placementSettings.SurvivalPreferences = stringVal
default:
return errors.Trace(errors.New("unknown placement policy option"))
}
Expand Down
46 changes: 45 additions & 1 deletion ddl/placement/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
)

// Refer to https://github.com/tikv/pd/issues/2701 .
Expand Down Expand Up @@ -123,7 +124,13 @@ func NewBundleFromConstraintsOptions(options *model.PlacementSettings) (*Bundle,
rules = append(rules, rule)
}
}

labels, err := newLocationLabelsFromSurvivalPreferences(options.SurvivalPreferences)
if err != nil {
return nil, err
}
for _, rule := range rules {
rule.LocationLabels = labels
}
return &Bundle{Rules: rules}, nil
}

Expand Down Expand Up @@ -155,9 +162,17 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error

var rules []*Rule

locationLabels, err := newLocationLabelsFromSurvivalPreferences(options.SurvivalPreferences)
if err != nil {
return nil, err
}

// in case empty primaryRegion and regions, just return an empty bundle
if primaryRegion == "" && len(regions) == 0 {
rules = append(rules, NewRule(Voter, followers+1, NewConstraintsDirect()))
for _, rule := range rules {
rule.LocationLabels = locationLabels
}
return &Bundle{Rules: rules}, nil
}

Expand Down Expand Up @@ -195,6 +210,11 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error
}
}

// set location labels
for _, rule := range rules {
rule.LocationLabels = locationLabels
}

return &Bundle{Rules: rules}, nil
}

Expand Down Expand Up @@ -223,6 +243,19 @@ func newBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err
return bundle, err
}

// newLocationLabelsFromSurvivalPreferences will parse the survival preferences into location labels.
func newLocationLabelsFromSurvivalPreferences(survivalPreferenceStr string) ([]string, error) {
if len(survivalPreferenceStr) > 0 {
labels := []string{}
err := yaml.UnmarshalStrict([]byte(survivalPreferenceStr), &labels)
if err != nil {
return nil, ErrInvalidSurvivalPreferenceFormat
}
return labels, nil
}
return nil, nil
}

// NewBundleFromOptions will transform options into the bundle.
func NewBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err error) {
bundle, err = newBundleFromOptions(options)
Expand Down Expand Up @@ -257,6 +290,15 @@ func (b *Bundle) String() string {
func (b *Bundle) Tidy() error {
extraCnt := map[PeerRoleType]int{}
newRules := b.Rules[:0]

// One Bundle is from one PlacementSettings, rule share same location labels, so we can use the first rule's location labels.
var locationLabels []string
for _, rule := range b.Rules {
if len(rule.LocationLabels) > 0 {
locationLabels = rule.LocationLabels
break
}
}
for i, rule := range b.Rules {
// useless Rule
if rule.Count <= 0 {
Expand Down Expand Up @@ -300,6 +342,8 @@ func (b *Bundle) Tidy() error {
Key: EngineLabelKey,
Values: []string{EngineLabelTiFlash},
}},
// the merged rule should have the same location labels with the original rules.
LocationLabels: locationLabels,
})
}
b.Rules = newRules
Expand Down
4 changes: 4 additions & 0 deletions ddl/placement/bundle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,9 @@ func TestTidy(t *testing.T) {
bundle.Rules = append(bundle.Rules, rules3...)
bundle.Rules = append(bundle.Rules, rules4...)

for _, r := range bundle.Rules {
r.LocationLabels = []string{"zone", "host"}
}
chkfunc := func() {
require.NoError(t, err)
require.Len(t, bundle.Rules, 3)
Expand All @@ -901,6 +904,7 @@ func TestTidy(t *testing.T) {
Values: []string{EngineLabelTiFlash},
},
}, bundle.Rules[2].Constraints)
require.Equal(t, []string{"zone", "host"}, bundle.Rules[2].LocationLabels)
}
err = bundle.Tidy()
chkfunc()
Expand Down
2 changes: 2 additions & 0 deletions ddl/placement/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ var (
ErrInvalidConstraintsMapcnt = errors.New("label constraints in map syntax have invalid replicas")
// ErrInvalidConstraintsFormat is from rule.go.
ErrInvalidConstraintsFormat = errors.New("invalid label constraints format")
// ErrInvalidSurvivalPreferenceFormat is from rule.go.
ErrInvalidSurvivalPreferenceFormat = errors.New("survival preference format should be in format [xxx=yyy, ...]")
// ErrInvalidConstraintsRelicas is from rule.go.
ErrInvalidConstraintsRelicas = errors.New("label constraints with invalid REPLICAS")
// ErrInvalidBundleID is from bundle.go.
Expand Down
19 changes: 10 additions & 9 deletions ddl/placement/rule.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,16 @@ type RuleGroupConfig struct {

// Rule is the core placement rule struct. Check https://github.com/tikv/pd/blob/master/server/schedule/placement/rule.go.
type Rule struct {
GroupID string `json:"group_id"`
ID string `json:"id"`
Index int `json:"index,omitempty"`
Override bool `json:"override,omitempty"`
StartKeyHex string `json:"start_key"`
EndKeyHex string `json:"end_key"`
Role PeerRoleType `json:"role"`
Count int `json:"count"`
Constraints Constraints `json:"label_constraints,omitempty"`
GroupID string `json:"group_id"`
ID string `json:"id"`
Index int `json:"index,omitempty"`
Override bool `json:"override,omitempty"`
StartKeyHex string `json:"start_key"`
EndKeyHex string `json:"end_key"`
Role PeerRoleType `json:"role"`
Count int `json:"count"`
Constraints Constraints `json:"label_constraints,omitempty"`
LocationLabels []string `json:"location_labels,omitempty"`
}

// TiFlashRule extends Rule with other necessary fields.
Expand Down
22 changes: 20 additions & 2 deletions ddl/placement_policy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ func TestPlacementPolicy(t *testing.T) {
"LEARNERS=1 " +
"LEARNER_CONSTRAINTS=\"[+region=cn-west-1]\" " +
"FOLLOWERS=3 " +
"FOLLOWER_CONSTRAINTS=\"[+disk=ssd]\"")
"FOLLOWER_CONSTRAINTS=\"[+disk=ssd]\"" +
"SURVIVAL_PREFERENCES=\"[region, zone]\"")

checkFunc := func(policyInfo *model.PolicyInfo) {
require.Equal(t, true, policyInfo.ID != 0)
Expand All @@ -168,6 +169,7 @@ func TestPlacementPolicy(t *testing.T) {
require.Equal(t, "[+region=cn-west-1]", policyInfo.LearnerConstraints)
require.Equal(t, model.StatePublic, policyInfo.State)
require.Equal(t, "", policyInfo.Schedule)
require.Equal(t, "[region, zone]", policyInfo.SurvivalPreferences)
}

// Check the policy is correctly reloaded in the information schema.
Expand Down Expand Up @@ -590,11 +592,16 @@ func TestCreateTableWithPlacementPolicy(t *testing.T) {
tk.MustExec("create placement policy x " +
"FOLLOWERS=2 " +
"CONSTRAINTS=\"[+disk=ssd]\" ")
tk.MustExec("create placement policy z " +
"FOLLOWERS=1 " +
"SURVIVAL_PREFERENCES=\"[region, zone]\"")
tk.MustExec("create placement policy y " +
"FOLLOWERS=3 " +
"CONSTRAINTS=\"[+region=bj]\" ")
tk.MustExec("create table t(a int)" +
"PLACEMENT POLICY=\"x\"")
tk.MustExec("create table tt(a int)" +
"PLACEMENT POLICY=\"z\"")
tk.MustQuery("SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TIDB_PLACEMENT_POLICY_NAME FROM information_schema.Tables WHERE TABLE_SCHEMA='test' AND TABLE_NAME = 't'").Check(testkit.Rows(`def test t x`))
tk.MustExec("create table t_range_p(id int) placement policy x partition by range(id) (" +
"PARTITION p0 VALUES LESS THAN (100)," +
Expand All @@ -617,7 +624,18 @@ func TestCreateTableWithPlacementPolicy(t *testing.T) {
require.Equal(t, "y", policyY.Name.L)
require.Equal(t, true, policyY.ID != 0)

tbl := external.GetTableByName(t, tk, "test", "t")
policyZ := testGetPolicyByName(t, tk.Session(), "z", true)
require.Equal(t, "z", policyZ.Name.L)
require.Equal(t, true, policyZ.ID != 0)
require.Equal(t, "[region, zone]", policyZ.SurvivalPreferences)

tbl := external.GetTableByName(t, tk, "test", "tt")
require.NotNil(t, tbl)
require.NotNil(t, tbl.Meta().PlacementPolicyRef)
require.Equal(t, "z", tbl.Meta().PlacementPolicyRef.Name.L)
require.Equal(t, policyZ.ID, tbl.Meta().PlacementPolicyRef.ID)

tbl = external.GetTableByName(t, tk, "test", "t")
require.NotNil(t, tbl)
require.NotNil(t, tbl.Meta().PlacementPolicyRef)
require.Equal(t, "x", tbl.Meta().PlacementPolicyRef.Name.L)
Expand Down
21 changes: 21 additions & 0 deletions docs/design/2020-06-24-placement-rules-in-sql.md
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,26 @@ If a table is imported when `tidb_placement_mode='IGNORE'`, and the placement po

The default value for `tidb_placement_mode` is `STRICT`. The option is an enum, and in future we may add support for a `WARN` mode.


#### survival preference
nolouch marked this conversation as resolved.
Show resolved Hide resolved

Some important data may need to store multiple copies across availability zones, so as to have high disaster recovery survivability, such as region-level survivability, `SURVIVAL_PREFERENCES` can provide survivability preference settings.

The following example sets a constraint that the data try to satisfy the Survival Preferences setting:

``` sql
CREATE PLACEMENT POLICY multiregion
follower=4
PRIMARY_REGION="region1"
SURVIVAL_PREFERENCES="[region, zone]";
```

For tables with this policy set, the data will first meet the survival goal of cross-region data isolation, and then ensure the survival goal of cross-zone data isolation.

> **Note:**
>
> `SURVIVAL_PREFERENCES` is equivalent to `LOCATION_LABELS` in PD. For more information, please refer to [Replica scheduling by topology label](https://docs.pingcap.com/tidb/dev/schedule-replicas-by-topology-labels#schedule-replicas-by-topology-labels).

#### Ambiguous and edge cases

The following two policies are not identical:
Expand Down Expand Up @@ -491,6 +511,7 @@ In this case the default rules will apply to placement, and the output from `SHO
- `FOLLOWER_CONSTRAINTS`
- `LEARNER_CONSTRAINTS`
- `PLACEMENT POLICY`
- `SURVIVAL_PREFERENCE`

For a more complex rule using partitions, consider the following example:

Expand Down
3 changes: 3 additions & 0 deletions domain/infosync/placement_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ import (
"path"
"sync"

"github.com/pingcap/log"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/util/pdapi"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)

// PlacementManager manages placement settings
Expand Down Expand Up @@ -72,6 +74,7 @@ func (m *PDPlacementManager) PutRuleBundles(ctx context.Context, bundles []*plac
return err
}

log.Debug("Put placement rule bundles", zap.String("rules", string(b)))
_, err = doRequest(ctx, "PutPlacementRules", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule")+"?partial=true", "POST", bytes.NewReader(b))
return err
}
Expand Down
5 changes: 5 additions & 0 deletions parser/ast/ddl.go
Original file line number Diff line number Diff line change
Expand Up @@ -2025,6 +2025,7 @@ const (
PlacementOptionLearnerConstraints
PlacementOptionFollowerConstraints
PlacementOptionVoterConstraints
PlacementOptionSurvivalPreferences
PlacementOptionPolicy
)

Expand Down Expand Up @@ -2089,6 +2090,10 @@ func (n *PlacementOption) Restore(ctx *format.RestoreCtx) error {
ctx.WriteKeyWord("PLACEMENT POLICY ")
ctx.WritePlain("= ")
ctx.WriteName(n.StrValue)
case PlacementOptionSurvivalPreferences:
ctx.WriteKeyWord("SURVIVAL_PREFERENCES ")
ctx.WritePlain("= ")
ctx.WriteString(n.StrValue)
default:
return errors.Errorf("invalid PlacementOption: %d", n.Tp)
}
Expand Down
1 change: 1 addition & 0 deletions parser/misc.go
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,7 @@ var tokenMap = map[string]int{
"SUBSTRING": substring,
"SUM": sum,
"SUPER": super,
"SURVIVAL_PREFERENCES": survivalPreferences,
"SWAPS": swaps,
"SWITCHES": switchesSym,
"SYSTEM": system,
Expand Down
1 change: 1 addition & 0 deletions parser/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -1736,6 +1736,7 @@ type PlacementSettings struct {
LearnerConstraints string `json:"learner_constraints"`
FollowerConstraints string `json:"follower_constraints"`
VoterConstraints string `json:"voter_constraints"`
SurvivalPreferences string `json:"survival_preferences"`
}

// PolicyInfo is the struct to store the placement policy.
Expand Down
Loading