Add send_to_leave_probability config options

chaos.go

@@ -132,7 +132,7 @@ func Bootstrap(cfg *config.Chaos) error {
 		}()
 	}
 
-	m.Start(cfg.Test.Seed, cfg.Test.OpsPerTick, func(tickIteration int) {
+	m.Start(func(tickIteration int) {
 		doSnapshot(snapshotters, sdb)
 		if cfg.Test.Convergence.Enabled && tickIteration%cfg.Test.Convergence.CheckEveryNTicks == 0 {
 			// TODO: stop the netsplit goroutine from spliting during this process.

config.demo.yaml

@@ -33,6 +33,10 @@ test:
   num_rooms: 2
   # The room version to test against. If empty, uses the default room version for the HS.
   # room_version: "11"
+  # number between 0-100 which is the % chance the user leaves the room instead of sending a message
+  # higher numbers cause more membership transitions overall which stresses the server in different ways.
+  # if 100, never sends messages.
+  send_to_leave_probability: 10
   # How many join/sends/leaves to do per tick.
   ops_per_tick: 50
   netsplits:

config.sample.yaml

@@ -42,6 +42,10 @@ test:
   num_rooms: 2
   # How many join/sends/leaves to do per tick.
   ops_per_tick: 50
+  # number between 0-100 which is the % chance the user leaves the room instead of sending a message
+  # higher numbers cause more membership transitions overall which stresses the server in different ways.
+  # if 100, never sends messages.
+  send_to_leave_probability: 10
   netsplits:
     # How long netsplits last. If 0, no netsplits.
     duration_secs: 4

config/config.go

@@ -19,12 +19,13 @@ type Chaos struct {
 }
 
 type TestConfig struct {
-	Seed        int64  `yaml:"seed"`
-	NumUsers    int    `yaml:"num_users"`
-	NumRooms    int    `yaml:"num_rooms"`
-	OpsPerTick  int    `yaml:"ops_per_tick"`
-	RoomVersion string `yaml:"room_version"`
-	Netsplits   struct {
+	Seed                   int64  `yaml:"seed"`
+	NumUsers               int    `yaml:"num_users"`
+	NumRooms               int    `yaml:"num_rooms"`
+	OpsPerTick             int    `yaml:"ops_per_tick"`
+	RoomVersion            string `yaml:"room_version"`
+	SendToLeaveProbability int    `yaml:"send_to_leave_probability"`
+	Netsplits              struct {
 		DurationSecs int `yaml:"duration_secs"`
 		FreeSecs     int `yaml:"free_secs"`
 	} `yaml:"netsplits"`

internal/master.go

@@ -133,9 +133,9 @@ func (m *Master) StartWorkers(numWorkers, opsPerTick int) {
 	}
 }
 
-func (m *Master) Start(seed int64, opsPerTick int, postTickFn func(tickIteration int)) {
+func (m *Master) Start(postTickFn func(tickIteration int)) {
 	userIDs := slices.Collect(maps.Keys(m.userIDToWorker))
-	stateMachine := NewStateMachine(seed, opsPerTick, userIDs, m.roomIDs)
+	stateMachine := NewStateMachine(m.cfg.Test.Seed, m.cfg.Test.OpsPerTick, m.cfg.Test.SendToLeaveProbability, userIDs, m.roomIDs)
 	convMasters := make([]CSAPIConvergence, len(m.masters))
 	for i := range convMasters {
 		convMasters[i] = &m.masters[i]

internal/state_machine.go

@@ -15,32 +15,37 @@ var (
 )
 
 type StateMachine struct {
-	Index            int
-	source           rand.Source
-	opsPerTick       int
-	userIDs          []string
-	roomIDs          []string
-	userToRoomStates map[string]map[string]State // user id => room id => state
+	Index                  int
+	source                 rand.Source
+	opsPerTick             int
+	userIDs                []string
+	roomIDs                []string
+	userToRoomStates       map[string]map[string]State // user id => room id => state
+	sendToleaveProbability int                         // 0-100 chance of leaving instead of sending a message
 }
 
-func NewStateMachine(seed int64, opsPerTick int, userIDs []string, roomIDs []string) *StateMachine {
+func NewStateMachine(seed int64, opsPerTick, sendToleaveProbability int, userIDs []string, roomIDs []string) *StateMachine {
 	userToRoomStates := make(map[string]map[string]State)
 	for _, u := range userIDs {
 		userToRoomStates[u] = make(map[string]State)
 		for _, r := range roomIDs {
 			userToRoomStates[u][r] = StateStart
 		}
 	}
+	if sendToleaveProbability < 0 || sendToleaveProbability > 100 {
+		panic("sendToleaveProbability must be between 0-100")
+	}
 
 	// ensure we get deterministic execution orders, we'll index into these arrays
 	slices.Sort(userIDs)
 	slices.Sort(roomIDs)
 	return &StateMachine{
-		source:           rand.NewSource(seed),
-		opsPerTick:       opsPerTick,
-		userToRoomStates: userToRoomStates,
-		userIDs:          userIDs,
-		roomIDs:          roomIDs,
+		source:                 rand.NewSource(seed),
+		opsPerTick:             opsPerTick,
+		userToRoomStates:       userToRoomStates,
+		userIDs:                userIDs,
+		roomIDs:                roomIDs,
+		sendToleaveProbability: sendToleaveProbability,
 	}
 }
 
@@ -72,8 +77,8 @@ func (s *StateMachine) Tick() []WorkerCommand {
 		case StateJoined:
 			fallthrough
 		case StateSend:
-			// either send a message or leave. We leave at 10% probability
-			shouldLeave := s.random(100)%10 == 0
+			// either send a message or leave. We leave at sendToleaveProbability
+			shouldLeave := s.random(100) < s.sendToleaveProbability
 			if shouldLeave {
 				cmds = append(cmds, WorkerCommand{
 					Action: ActionLeave,

internal/state_machine_test.go

@@ -13,7 +13,7 @@ func TestStateMachineOnlyDoesValidTransitions(t *testing.T) {
 		StateSend:   {StateSend, StateLeft},
 		StateStart:  {StateJoined},
 	}
-	sm := NewStateMachine(42, 10, []string{"alice", "bob"}, []string{"!foo", "!bar", "!baz"})
+	sm := NewStateMachine(42, 10, 10, []string{"alice", "bob"}, []string{"!foo", "!bar", "!baz"})
 
 	for i := 0; i < 100; i++ {
 		cmds := sm.Tick()
@@ -31,12 +31,12 @@ func TestStateMachineOnlyDoesValidTransitions(t *testing.T) {
 }
 
 func TestStateMachineIsDeterministic(t *testing.T) {
-	sm := NewStateMachine(42, 4, []string{"alice", "bob"}, []string{"!foo", "!bar", "!baz"})
+	sm := NewStateMachine(42, 4, 10, []string{"alice", "bob"}, []string{"!foo", "!bar", "!baz"})
 	cmds := sm.Tick()
 	sm.Apply(cmds)
 	want := sm.userToRoomStates
 	for i := 0; i < 100; i++ {
-		sm := NewStateMachine(42, 4, []string{"bob", "alice"}, []string{"!foo", "!baz", "!bar"})
+		sm := NewStateMachine(42, 4, 10, []string{"bob", "alice"}, []string{"!foo", "!baz", "!bar"})
 		cmds := sm.Tick()
 		sm.Apply(cmds)
 		if !reflect.DeepEqual(sm.userToRoomStates, want) {