diff --git a/internal/armada/mocks/mock_repository.go b/internal/armada/mocks/mock_repository.go index d908290bbe8..a9f79e2702b 100644 --- a/internal/armada/mocks/mock_repository.go +++ b/internal/armada/mocks/mock_repository.go @@ -87,6 +87,22 @@ func (m *MockQueueRepository) GetQueue(arg0 *armadacontext.Context, arg1 string) return ret0, ret1 } +// CordonQueue mocks base method. +func (m *MockQueueRepository) CordonQueue(arg0 *armadacontext.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CordonQueue", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UncordonQueue mocks base method. +func (m *MockQueueRepository) UncordonQueue(arg0 *armadacontext.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UncordonQueue", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + // GetQueue indicates an expected call of GetQueue. func (mr *MockQueueRepositoryMockRecorder) GetQueue(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() @@ -106,3 +122,15 @@ func (mr *MockQueueRepositoryMockRecorder) UpdateQueue(arg0, arg1 interface{}) * mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateQueue", reflect.TypeOf((*MockQueueRepository)(nil).UpdateQueue), arg0, arg1) } + +// CordonQueue indicates an expected call of CordonQueue. +func (mr *MockQueueRepositoryMockRecorder) CordonQueue(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CordonQueue", reflect.TypeOf((*MockQueueRepository)(nil).CordonQueue), arg0, arg1) +} + +// UncordonQueue indicates an expected call of UncordonQueue. +func (mr *MockQueueRepositoryMockRecorder) UncordonQueue(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UncordonQueue", reflect.TypeOf((*MockQueueRepository)(nil).UncordonQueue), arg0, arg1) +} diff --git a/internal/armada/permissions/permissions.go b/internal/armada/permissions/permissions.go index cab12193c8a..f2ad819fb17 100644 --- a/internal/armada/permissions/permissions.go +++ b/internal/armada/permissions/permissions.go @@ -13,5 +13,6 @@ const ( WatchAllEvents = "watch_all_events" CreateQueue = "create_queue" DeleteQueue = "delete_queue" + CordonQueue = "cordon_queue" CordonNodes = "cordon_nodes" ) diff --git a/internal/armada/queue/queue_repository.go b/internal/armada/queue/queue_repository.go index 7224e656028..68029219571 100644 --- a/internal/armada/queue/queue_repository.go +++ b/internal/armada/queue/queue_repository.go @@ -35,6 +35,8 @@ type QueueRepository interface { CreateQueue(*armadacontext.Context, queue.Queue) error UpdateQueue(*armadacontext.Context, queue.Queue) error DeleteQueue(ctx *armadacontext.Context, name string) error + CordonQueue(ctx *armadacontext.Context, name string) error + UncordonQueue(ctx *armadacontext.Context, name string) error } type ReadOnlyQueueRepository interface { @@ -116,6 +118,24 @@ func (r *PostgresQueueRepository) DeleteQueue(ctx *armadacontext.Context, name s return nil } +func (r *PostgresQueueRepository) CordonQueue(ctx *armadacontext.Context, name string) error { + queueToCordon, err := r.GetQueue(ctx, name) + if err != nil { + return err + } + queueToCordon.Cordoned = true + return r.upsertQueue(ctx, queueToCordon) +} + +func (r *PostgresQueueRepository) UncordonQueue(ctx *armadacontext.Context, name string) error { + queueToUncordon, err := r.GetQueue(ctx, name) + if err != nil { + return err + } + queueToUncordon.Cordoned = false + return r.upsertQueue(ctx, queueToUncordon) +} + func (r *PostgresQueueRepository) upsertQueue(ctx *armadacontext.Context, queue queue.Queue) error { data, err := proto.Marshal(queue.ToAPI()) if err != nil { diff --git a/internal/armada/queue/queue_service.go b/internal/armada/queue/queue_service.go index 65cad8a6977..de4861c5922 100644 --- a/internal/armada/queue/queue_service.go +++ b/internal/armada/queue/queue_service.go @@ -2,6 +2,7 @@ package queue import ( "context" + "fmt" "math" "github.com/gogo/protobuf/types" @@ -37,9 +38,9 @@ func (s *Server) CreateQueue(grpcCtx context.Context, req *api.Queue) (*types.Em err := s.authorizer.AuthorizeAction(ctx, permissions.CreateQueue) var ep *armadaerrors.ErrUnauthorized if errors.As(err, &ep) { - return nil, status.Errorf(codes.PermissionDenied, "[CreateQueue] error creating queue %s: %s", req.Name, ep) + return nil, status.Errorf(codes.PermissionDenied, "error creating queue %s: %s", req.Name, ep) } else if err != nil { - return nil, status.Errorf(codes.Unavailable, "[CreateQueue] error checking permissions: %s", err) + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) } if len(req.UserOwners) == 0 { @@ -49,15 +50,15 @@ func (s *Server) CreateQueue(grpcCtx context.Context, req *api.Queue) (*types.Em queue, err := queue.NewQueue(req) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "[CreateQueue] error validating queue: %s", err) + return nil, status.Errorf(codes.InvalidArgument, "error validating queue: %s", err) } err = s.queueRepository.CreateQueue(ctx, queue) var eq *ErrQueueAlreadyExists if errors.As(err, &eq) { - return nil, status.Errorf(codes.AlreadyExists, "[CreateQueue] error creating queue: %s", err) + return nil, status.Errorf(codes.AlreadyExists, "error creating queue: %s", err) } else if err != nil { - return nil, status.Errorf(codes.Unavailable, "[CreateQueue] error creating queue: %s", err) + return nil, status.Errorf(codes.Unavailable, "error creating queue: %s", err) } return &types.Empty{}, nil @@ -87,22 +88,22 @@ func (s *Server) UpdateQueue(grpcCtx context.Context, req *api.Queue) (*types.Em err := s.authorizer.AuthorizeAction(ctx, permissions.CreateQueue) var ep *armadaerrors.ErrUnauthorized if errors.As(err, &ep) { - return nil, status.Errorf(codes.PermissionDenied, "[UpdateQueue] error updating queue %s: %s", req.Name, ep) + return nil, status.Errorf(codes.PermissionDenied, "error updating queue %s: %s", req.Name, ep) } else if err != nil { - return nil, status.Errorf(codes.Unavailable, "[UpdateQueue] error checking permissions: %s", err) + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) } queue, err := queue.NewQueue(req) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "[UpdateQueue] error: %s", err) + return nil, status.Errorf(codes.InvalidArgument, "error: %s", err) } err = s.queueRepository.UpdateQueue(ctx, queue) var e *ErrQueueNotFound if errors.As(err, &e) { - return nil, status.Errorf(codes.NotFound, "[UpdateQueue] error: %s", err) + return nil, status.Errorf(codes.NotFound, "error: %s", err) } else if err != nil { - return nil, status.Errorf(codes.Unavailable, "[UpdateQueue] error getting queue %q: %s", queue.Name, err) + return nil, status.Errorf(codes.Unavailable, "error getting queue %q: %s", queue.Name, err) } return &types.Empty{}, nil @@ -133,13 +134,13 @@ func (s *Server) DeleteQueue(grpcCtx context.Context, req *api.QueueDeleteReques err := s.authorizer.AuthorizeAction(ctx, permissions.DeleteQueue) var ep *armadaerrors.ErrUnauthorized if errors.As(err, &ep) { - return nil, status.Errorf(codes.PermissionDenied, "[DeleteQueue] error deleting queue %s: %s", req.Name, ep) + return nil, status.Errorf(codes.PermissionDenied, "error deleting queue %s: %s", req.Name, ep) } else if err != nil { - return nil, status.Errorf(codes.Unavailable, "[DeleteQueue] error checking permissions: %s", err) + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) } err = s.queueRepository.DeleteQueue(ctx, req.Name) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "[DeleteQueue] error deleting queue %s: %s", req.Name, err) + return nil, status.Errorf(codes.InvalidArgument, "error deleting queue %s: %s", req.Name, err) } return &types.Empty{}, nil } @@ -149,9 +150,9 @@ func (s *Server) GetQueue(grpcCtx context.Context, req *api.QueueGetRequest) (*a queue, err := s.queueRepository.GetQueue(ctx, req.Name) var e *ErrQueueNotFound if errors.As(err, &e) { - return nil, status.Errorf(codes.NotFound, "[GetQueue] error: %s", err) + return nil, status.Errorf(codes.NotFound, "error: %s", err) } else if err != nil { - return nil, status.Errorf(codes.Unavailable, "[GetQueue] error getting queue %q: %s", req.Name, err) + return nil, status.Errorf(codes.Unavailable, "error getting queue %q: %s", req.Name, err) } return queue.ToAPI(), nil } @@ -189,3 +190,41 @@ func (s *Server) GetQueues(req *api.StreamingQueueGetRequest, stream api.QueueSe } return nil } + +func (s *Server) CordonQueue(grpcCtx context.Context, req *api.QueueCordonRequest) (*types.Empty, error) { + ctx := armadacontext.FromGrpcCtx(grpcCtx) + + err := s.authorizer.AuthorizeAction(ctx, permissions.CordonQueue) + var ep *armadaerrors.ErrUnauthorized + if errors.As(err, &ep) { + return nil, status.Errorf(codes.PermissionDenied, "error cordoning queue %s: %s", req.Name, ep) + } else if err != nil { + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) + } + + queueName := req.Name + if queueName == "" { + return nil, fmt.Errorf("cannot cordon queue with empty name") + } + + return &types.Empty{}, s.queueRepository.CordonQueue(ctx, queueName) +} + +func (s *Server) UncordonQueue(grpcCtx context.Context, req *api.QueueUncordonRequest) (*types.Empty, error) { + ctx := armadacontext.FromGrpcCtx(grpcCtx) + + err := s.authorizer.AuthorizeAction(ctx, permissions.CordonQueue) + var ep *armadaerrors.ErrUnauthorized + if errors.As(err, &ep) { + return nil, status.Errorf(codes.PermissionDenied, "error uncordoning queue %s: %s", req.Name, ep) + } else if err != nil { + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) + } + + queueName := req.Name + if queueName == "" { + return nil, fmt.Errorf("cannot uncordon queue with empty name") + } + + return &types.Empty{}, s.queueRepository.UncordonQueue(ctx, queueName) +} diff --git a/internal/armadactl/app.go b/internal/armadactl/app.go index 2c445c5962e..2d6f351b49b 100644 --- a/internal/armadactl/app.go +++ b/internal/armadactl/app.go @@ -47,10 +47,13 @@ type Params struct { // However, they are user-replaceable to facilitate testing. // TODO Consider replacing with an interface type QueueAPI struct { - Create queue.CreateAPI - Delete queue.DeleteAPI - Get queue.GetAPI - Update queue.UpdateAPI + Create queue.CreateAPI + Delete queue.DeleteAPI + Get queue.GetAPI + GetAll queue.GetAllAPI + Update queue.UpdateAPI + Cordon queue.CordonAPI + Uncordon queue.UncordonAPI } // New instantiates an App with default parameters, including standard output diff --git a/internal/scheduler/constraints/constraints.go b/internal/scheduler/constraints/constraints.go index 5a8c392a872..3de4d95e2d4 100644 --- a/internal/scheduler/constraints/constraints.go +++ b/internal/scheduler/constraints/constraints.go @@ -23,6 +23,7 @@ const ( // Indicates that the scheduling rate limit has been exceeded. GlobalRateLimitExceededUnschedulableReason = "global scheduling rate limit exceeded" QueueRateLimitExceededUnschedulableReason = "queue scheduling rate limit exceeded" + SchedulingPausedOnQueueUnschedulableReason = "scheduling paused on queue" // Indicates that scheduling a gang would exceed the rate limit. GlobalRateLimitExceededByGangUnschedulableReason = "gang would exceed global scheduling rate limit" @@ -54,7 +55,7 @@ func IsTerminalUnschedulableReason(reason string) bool { // IsTerminalQueueUnschedulableReason returns true if reason indicates // it's not possible to schedule any more jobs from this queue in this round. func IsTerminalQueueUnschedulableReason(reason string) bool { - return reason == QueueRateLimitExceededUnschedulableReason + return reason == QueueRateLimitExceededUnschedulableReason || reason == SchedulingPausedOnQueueUnschedulableReason } // SchedulingConstraints contains scheduling constraints, e.g., per-queue resource limits. @@ -75,6 +76,8 @@ type SchedulingConstraints struct { type queueSchedulingConstraints struct { // Scheduling constraints by priority class. PriorityClassSchedulingConstraintsByPriorityClassName map[string]priorityClassSchedulingConstraints + // Determines whether scheduling has been paused for this queue + Cordoned bool } // priorityClassSchedulingConstraints contains scheduling constraints that apply to jobs of a specific priority class. @@ -84,12 +87,7 @@ type priorityClassSchedulingConstraints struct { MaximumResourcesPerQueue map[string]resource.Quantity } -func NewSchedulingConstraints( - pool string, - totalResources schedulerobjects.ResourceList, - config configuration.SchedulingConfig, - queues []*api.Queue, -) SchedulingConstraints { +func NewSchedulingConstraints(pool string, totalResources schedulerobjects.ResourceList, config configuration.SchedulingConfig, queues []*api.Queue, cordonStatusByQueue map[string]bool) SchedulingConstraints { priorityClassSchedulingConstraintsByPriorityClassName := make(map[string]priorityClassSchedulingConstraints, len(config.PriorityClasses)) for name, priorityClass := range config.PriorityClasses { maximumResourceFractionPerQueue := priorityClass.MaximumResourceFractionPerQueue @@ -117,10 +115,9 @@ func NewSchedulingConstraints( MaximumResourcesPerQueue: absoluteFromRelativeLimits(totalResources.Resources, maximumResourceFraction), } } - if len(priorityClassSchedulingConstraintsByPriorityClassNameForQueue) > 0 { - queueSchedulingConstraintsByQueueName[queue.Name] = queueSchedulingConstraints{ - PriorityClassSchedulingConstraintsByPriorityClassName: priorityClassSchedulingConstraintsByPriorityClassNameForQueue, - } + queueSchedulingConstraintsByQueueName[queue.Name] = queueSchedulingConstraints{ + PriorityClassSchedulingConstraintsByPriorityClassName: priorityClassSchedulingConstraintsByPriorityClassNameForQueue, + Cordoned: cordonStatusByQueue[queue.Name], } } @@ -182,6 +179,9 @@ func (constraints *SchedulingConstraints) CheckConstraints( return false, GlobalRateLimitExceededByGangUnschedulableReason, nil } + if queueConstraints, ok := constraints.queueSchedulingConstraintsByQueueName[qctx.Queue]; ok && queueConstraints.Cordoned { + return false, SchedulingPausedOnQueueUnschedulableReason, nil + } // Per-queue rate limiter check. tokens = qctx.Limiter.TokensAt(sctx.Started) if tokens <= 0 { diff --git a/internal/scheduler/constraints/constraints_test.go b/internal/scheduler/constraints/constraints_test.go index 4d749827f6c..497083c5b1e 100644 --- a/internal/scheduler/constraints/constraints_test.go +++ b/internal/scheduler/constraints/constraints_test.go @@ -28,94 +28,67 @@ type constraintTest struct { func TestConstraints(t *testing.T) { tests := map[string]*constraintTest{ - "no-constraints": makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - makeSchedulingConfig(), - []*api.Queue{}, - )), - "empty-queue-constraints": makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - makeSchedulingConfig(), - []*api.Queue{{Name: "queue-1", ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{}}}, - )), - "within-constraints": makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ - MaximumResourceFractionToSchedule: map[string]float64{"cpu": 0.1, "memory": 0.1}, - MaxQueueLookback: 1000, - PriorityClasses: map[string]types.PriorityClass{"priority-class-1": {MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{"pool-1": {"cpu": 0.9, "memory": 0.9}}}}, - }, - []*api.Queue{{Name: "queue-1", ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{"priority-class-1": {MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}}}}}, - )), - "exceeds-queue-priority-class-constraint": func() *constraintTest { - t := makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), + "no-constraints": makeConstraintsTest( + NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), makeSchedulingConfig(), - []*api.Queue{ - { - Name: "queue-1", - ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ - "priority-class-1": { - MaximumResourceFraction: map[string]float64{"cpu": 0.000001, "memory": 0.9}, - }, + []*api.Queue{}, + map[string]bool{})), + "empty-queue-constraints": makeConstraintsTest( + NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), + makeSchedulingConfig(), + []*api.Queue{{Name: "queue-1", ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{}}}, + map[string]bool{"queue-1": false})), + "within-constraints": makeConstraintsTest(NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ + MaximumResourceFractionToSchedule: map[string]float64{"cpu": 0.1, "memory": 0.1}, + MaxQueueLookback: 1000, + PriorityClasses: map[string]types.PriorityClass{"priority-class-1": {MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{"pool-1": {"cpu": 0.9, "memory": 0.9}}}}, + }, []*api.Queue{{Name: "queue-1", ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{"priority-class-1": {MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}}}}}, map[string]bool{"queue-1": false})), + "exceeds-queue-priority-class-constraint": func() *constraintTest { + t := makeConstraintsTest(NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), makeSchedulingConfig(), []*api.Queue{ + { + Name: "queue-1", + ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ + "priority-class-1": { + MaximumResourceFraction: map[string]float64{"cpu": 0.000001, "memory": 0.9}, }, }, }, - )) + }, map[string]bool{"queue-1": false})) t.expectedCheckConstraintsReason = "resource limit exceeded" return t }(), "exceeds-queue-priority-class-pool-constraint": func() *constraintTest { - t := makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - makeSchedulingConfig(), - []*api.Queue{ - { - Name: "queue-1", - ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ - "priority-class-1": { - MaximumResourceFractionByPool: map[string]*api.PriorityClassPoolResourceLimits{ - "pool-1": { - MaximumResourceFraction: map[string]float64{"cpu": 0.000001, "memory": 0.9}, - }, + t := makeConstraintsTest(NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), makeSchedulingConfig(), []*api.Queue{ + { + Name: "queue-1", + ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ + "priority-class-1": { + MaximumResourceFractionByPool: map[string]*api.PriorityClassPoolResourceLimits{ + "pool-1": { + MaximumResourceFraction: map[string]float64{"cpu": 0.000001, "memory": 0.9}, }, }, }, }, }, - )) + }, map[string]bool{"queue-1": false})) t.expectedCheckConstraintsReason = "resource limit exceeded" return t }(), "exceeds-priority-class-constraint": func() *constraintTest { - t := makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ - MaximumResourceFractionToSchedule: map[string]float64{"cpu": 0.1, "memory": 0.1}, - MaxQueueLookback: 1000, - PriorityClasses: map[string]types.PriorityClass{"priority-class-1": {MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{"pool-1": {"cpu": 0.00000001, "memory": 0.9}}}}, - }, - []*api.Queue{}, - )) - t.expectedCheckConstraintsReason = "resource limit exceeded" - return t - }(), - "priority-class-constraint-ignored-if-there-is-a-queue-constraint": makeConstraintsTest(NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ + t := makeConstraintsTest(NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ MaximumResourceFractionToSchedule: map[string]float64{"cpu": 0.1, "memory": 0.1}, MaxQueueLookback: 1000, PriorityClasses: map[string]types.PriorityClass{"priority-class-1": {MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{"pool-1": {"cpu": 0.00000001, "memory": 0.9}}}}, - }, - []*api.Queue{{Name: "queue-1", ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{"priority-class-1": {MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}}}}}, - )), + }, []*api.Queue{}, map[string]bool{})) + t.expectedCheckConstraintsReason = "resource limit exceeded" + return t + }(), + "priority-class-constraint-ignored-if-there-is-a-queue-constraint": makeConstraintsTest(NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ + MaximumResourceFractionToSchedule: map[string]float64{"cpu": 0.1, "memory": 0.1}, + MaxQueueLookback: 1000, + PriorityClasses: map[string]types.PriorityClass{"priority-class-1": {MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{"pool-1": {"cpu": 0.00000001, "memory": 0.9}}}}, + }, []*api.Queue{{Name: "queue-1", ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{"priority-class-1": {MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}}}}}, nil)), "one-constraint-per-level-falls-back-as-expected--within-limits": makeMultiLevelConstraintsTest( map[string]resource.Quantity{"a": resource.MustParse("99"), "b": resource.MustParse("19"), "c": resource.MustParse("2.9"), "d": resource.MustParse("0.39")}, "", @@ -165,109 +138,84 @@ func TestCapResources(t *testing.T) { expectedResources schedulerobjects.QuantityByTAndResourceType[string] }{ "no contraints": { - constraints: NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - makeSchedulingConfig(), - []*api.Queue{}, - ), + constraints: NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), makeSchedulingConfig(), []*api.Queue{}, map[string]bool{}), queue: "queue-1", resources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("1000", "1000Gi")}, expectedResources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("1000", "1000Gi")}, }, "unconstrained": { - constraints: NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ - PriorityClasses: map[string]types.PriorityClass{ - "priority-class-1": { - MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ - "pool-1": {"cpu": 0.1, "memory": 0.9}, - }, + constraints: NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ + PriorityClasses: map[string]types.PriorityClass{ + "priority-class-1": { + MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ + "pool-1": {"cpu": 0.1, "memory": 0.9}, }, }, }, - []*api.Queue{}, - ), + }, []*api.Queue{}, map[string]bool{}), queue: "queue-1", resources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("1", "1Gi")}, expectedResources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("1", "1Gi")}, }, "per pool cap": { - constraints: NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ - PriorityClasses: map[string]types.PriorityClass{ - "priority-class-1": { - MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ - "pool-1": {"cpu": 0.1, "memory": 0.9}, - }, + constraints: NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ + PriorityClasses: map[string]types.PriorityClass{ + "priority-class-1": { + MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ + "pool-1": {"cpu": 0.1, "memory": 0.9}, }, }, }, - []*api.Queue{}, - ), + }, []*api.Queue{}, map[string]bool{}), queue: "queue-1", resources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("1000", "1000Gi")}, expectedResources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("100", "900Gi")}, }, "per queue cap": { - constraints: NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ - PriorityClasses: map[string]types.PriorityClass{ - "priority-class-1": { - MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ - "pool-1": {"cpu": 0.1, "memory": 0.9}, - }, + constraints: NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ + PriorityClasses: map[string]types.PriorityClass{ + "priority-class-1": { + MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ + "pool-1": {"cpu": 0.1, "memory": 0.9}, }, }, }, - []*api.Queue{ - { - Name: "queue-1", - ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ - "priority-class-1": { - MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}, - }, + }, []*api.Queue{ + { + Name: "queue-1", + ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ + "priority-class-1": { + MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}, }, }, }, - ), + }, map[string]bool{}), queue: "queue-1", resources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("1000", "1000Gi")}, expectedResources: map[string]schedulerobjects.ResourceList{"priority-class-1": makeResourceList("900", "900Gi")}, }, "per queue cap with multi pc": { - constraints: NewSchedulingConstraints( - "pool-1", - makeResourceList("1000", "1000Gi"), - configuration.SchedulingConfig{ - PriorityClasses: map[string]types.PriorityClass{ - "priority-class-1": { - MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ - "pool-1": {"cpu": 0.1, "memory": 0.9}, - }, + constraints: NewSchedulingConstraints("pool-1", makeResourceList("1000", "1000Gi"), configuration.SchedulingConfig{ + PriorityClasses: map[string]types.PriorityClass{ + "priority-class-1": { + MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ + "pool-1": {"cpu": 0.1, "memory": 0.9}, }, }, }, - []*api.Queue{ - { - Name: "queue-1", - ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ - "priority-class-1": { - MaximumResourceFraction: map[string]float64{"cpu": 0.1, "memory": 0.1}, - }, - "priority-class-2": { - MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}, - }, + }, []*api.Queue{ + { + Name: "queue-1", + ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ + "priority-class-1": { + MaximumResourceFraction: map[string]float64{"cpu": 0.1, "memory": 0.1}, + }, + "priority-class-2": { + MaximumResourceFraction: map[string]float64{"cpu": 0.9, "memory": 0.9}, }, }, }, - ), + }, map[string]bool{}), queue: "queue-1", resources: map[string]schedulerobjects.ResourceList{ "priority-class-1": makeResourceList("1000", "1000Gi"), @@ -340,40 +288,42 @@ func makeMultiLevelConstraintsTest(requirements map[string]resource.Quantity, ex } func makeMultiLevelConstraints() SchedulingConstraints { - return NewSchedulingConstraints( - "pool-1", - schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"a": resource.MustParse("1000"), "b": resource.MustParse("1000"), "c": resource.MustParse("1000"), "d": resource.MustParse("1000")}}, - configuration.SchedulingConfig{ - MaxQueueLookback: 1000, - PriorityClasses: map[string]types.PriorityClass{ - "priority-class-1": { - MaximumResourceFractionPerQueue: map[string]float64{ - "a": 0.0001, "b": 0.0002, "c": 0.0003, "d": 0.0004, - }, - MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ - "pool-1": { - "a": 0.001, "b": 0.002, "c": 0.003, - }, + return NewSchedulingConstraints("pool-1", schedulerobjects.ResourceList{ + Resources: map[string]resource.Quantity{ + "a": resource.MustParse("1000"), + "b": resource.MustParse("1000"), + "c": resource.MustParse("1000"), + "d": resource.MustParse("1000"), + }, + }, configuration.SchedulingConfig{ + MaxQueueLookback: 1000, + PriorityClasses: map[string]types.PriorityClass{ + "priority-class-1": { + MaximumResourceFractionPerQueue: map[string]float64{ + "a": 0.0001, "b": 0.0002, "c": 0.0003, "d": 0.0004, + }, + MaximumResourceFractionPerQueueByPool: map[string]map[string]float64{ + "pool-1": { + "a": 0.001, "b": 0.002, "c": 0.003, }, }, }, }, - []*api.Queue{ - { - Name: "queue-1", - ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ - "priority-class-1": { - MaximumResourceFraction: map[string]float64{"a": 0.01, "b": 0.02}, - MaximumResourceFractionByPool: map[string]*api.PriorityClassPoolResourceLimits{ - "pool-1": { - MaximumResourceFraction: map[string]float64{"a": 0.1}, - }, + }, []*api.Queue{ + { + Name: "queue-1", + ResourceLimitsByPriorityClassName: map[string]*api.PriorityClassResourceLimits{ + "priority-class-1": { + MaximumResourceFraction: map[string]float64{"a": 0.01, "b": 0.02}, + MaximumResourceFractionByPool: map[string]*api.PriorityClassPoolResourceLimits{ + "pool-1": { + MaximumResourceFraction: map[string]float64{"a": 0.1}, }, }, }, }, }, - ) + }, map[string]bool{}) } func TestScaleQuantity(t *testing.T) { diff --git a/internal/scheduler/database/migrations/013_drop_queues_table.sql b/internal/scheduler/database/migrations/013_drop_queues_table.sql new file mode 100644 index 00000000000..efd0830e998 --- /dev/null +++ b/internal/scheduler/database/migrations/013_drop_queues_table.sql @@ -0,0 +1 @@ +DROP TABLE queues; diff --git a/internal/scheduler/gang_scheduler_test.go b/internal/scheduler/gang_scheduler_test.go index 5be1dbf06b4..b78beae55b6 100644 --- a/internal/scheduler/gang_scheduler_test.go +++ b/internal/scheduler/gang_scheduler_test.go @@ -640,12 +640,7 @@ func TestGangScheduler(t *testing.T) { ) require.NoError(t, err) } - constraints := schedulerconstraints.NewSchedulingConstraints( - "pool", - tc.TotalResources, - tc.SchedulingConfig, - nil, - ) + constraints := schedulerconstraints.NewSchedulingConstraints("pool", tc.TotalResources, tc.SchedulingConfig, nil, map[string]bool{}) floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(tc.SchedulingConfig.ExperimentalFloatingResources) require.NoError(t, err) sch, err := NewGangScheduler(sctx, constraints, floatingResourceTypes, nodeDb) diff --git a/internal/scheduler/preempting_queue_scheduler_test.go b/internal/scheduler/preempting_queue_scheduler_test.go index aff13ae00fb..dde7f7c5144 100644 --- a/internal/scheduler/preempting_queue_scheduler_test.go +++ b/internal/scheduler/preempting_queue_scheduler_test.go @@ -1854,12 +1854,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { ) require.NoError(t, err) } - constraints := schedulerconstraints.NewSchedulingConstraints( - "pool", - tc.TotalResources, - tc.SchedulingConfig, - nil, - ) + constraints := schedulerconstraints.NewSchedulingConstraints("pool", tc.TotalResources, tc.SchedulingConfig, nil, map[string]bool{}) sctx.UpdateFairShares() sch := NewPreemptingQueueScheduler( sctx, @@ -2207,12 +2202,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { schedulerobjects.NewResourceList(0), schedulerobjects.NewResourceList(0), limiterByQueue[queue]) require.NoError(b, err) } - constraints := schedulerconstraints.NewSchedulingConstraints( - "pool", - nodeDb.TotalResources(), - tc.SchedulingConfig, - nil, - ) + constraints := schedulerconstraints.NewSchedulingConstraints("pool", nodeDb.TotalResources(), tc.SchedulingConfig, nil, map[string]bool{}) sch := NewPreemptingQueueScheduler( sctx, constraints, diff --git a/internal/scheduler/queue_scheduler_test.go b/internal/scheduler/queue_scheduler_test.go index af45614b09e..f4bc44e3a70 100644 --- a/internal/scheduler/queue_scheduler_test.go +++ b/internal/scheduler/queue_scheduler_test.go @@ -534,12 +534,7 @@ func TestQueueScheduler(t *testing.T) { ) require.NoError(t, err) } - constraints := schedulerconstraints.NewSchedulingConstraints( - "pool", - tc.TotalResources, - tc.SchedulingConfig, - tc.Queues, - ) + constraints := schedulerconstraints.NewSchedulingConstraints("pool", tc.TotalResources, tc.SchedulingConfig, tc.Queues, map[string]bool{}) jobIteratorByQueue := make(map[string]JobIterator) for _, q := range tc.Queues { it := jobRepo.GetJobIterator(q.Name) diff --git a/internal/scheduler/scheduling_algo.go b/internal/scheduler/scheduling_algo.go index c0fc52fe090..4413bae2758 100644 --- a/internal/scheduler/scheduling_algo.go +++ b/internal/scheduler/scheduling_algo.go @@ -223,6 +223,7 @@ type fairSchedulingAlgoContext struct { jobIdsByGangId map[string]map[string]bool gangIdByJobId map[string]string allocationByPoolAndQueueAndPriorityClass map[string]map[string]schedulerobjects.QuantityByTAndResourceType[string] + cordonStatusByQueue map[string]bool executors []*schedulerobjects.Executor txn *jobdb.Txn } @@ -234,6 +235,7 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con executorById := map[string]*schedulerobjects.Executor{} nodesByPoolAndExecutor := map[string]map[string][]*schedulerobjects.Node{} allKnownPools := map[string]bool{} + cordonStatusByQueue := make(map[string]bool) for _, executor := range executors { executorById[executor.Id] = executor @@ -266,6 +268,7 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con priorityFactorByQueue := make(map[string]float64) for _, queue := range queues { priorityFactorByQueue[queue.Name] = float64(queue.PriorityFactor) + cordonStatusByQueue[queue.Name] = queue.Cordoned } // Get the total capacity available across executors. @@ -319,12 +322,15 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con queueResources = schedulerobjects.QuantityByTAndResourceType[string]{} poolQueueResources[job.Queue()] = queueResources } - pcResources, ok := queueResources[job.PriorityClassName()] - if !ok { - pcResources = schedulerobjects.NewResourceList(len(job.PodRequirements().ResourceRequirements.Requests)) - queueResources[job.PriorityClassName()] = pcResources + // Queued jobs should not be considered for paused queues, so demand := running + if !cordonStatusByQueue[job.Queue()] || !job.Queued() { + pcResources, ok := queueResources[job.PriorityClassName()] + if !ok { + pcResources = schedulerobjects.NewResourceList(len(job.PodRequirements().ResourceRequirements.Requests)) + queueResources[job.PriorityClassName()] = pcResources + } + pcResources.AddV1ResourceList(job.PodRequirements().ResourceRequirements.Requests) } - pcResources.AddV1ResourceList(job.PodRequirements().ResourceRequirements.Requests) } if job.Queued() { @@ -387,6 +393,7 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con nodeIdByJobId: nodeIdByJobId, jobIdsByGangId: jobIdsByGangId, gangIdByJobId: gangIdByJobId, + cordonStatusByQueue: cordonStatusByQueue, allocationByPoolAndQueueAndPriorityClass: totalAllocationByPoolAndQueue, executors: executors, txn: txn, @@ -438,19 +445,13 @@ func (l *FairSchedulingAlgo) schedulePool( totalResources, ) - constraints := schedulerconstraints.NewSchedulingConstraints( - pool, - fsctx.totalCapacityByPool[pool], - l.schedulingConfig, - fsctx.queues, - ) + constraints := schedulerconstraints.NewSchedulingConstraints(pool, fsctx.totalCapacityByPool[pool], l.schedulingConfig, fsctx.queues, fsctx.cordonStatusByQueue) demandByQueue, ok := fsctx.demandByPoolByQueue[pool] if !ok { demandByQueue = map[string]schedulerobjects.QuantityByTAndResourceType[string]{} } - now := time.Now() for queue, priorityFactor := range fsctx.priorityFactorByQueue { demand, hasDemand := demandByQueue[queue] if !hasDemand { @@ -478,8 +479,6 @@ func (l *FairSchedulingAlgo) schedulePool( l.limiterByQueue[queue] = queueLimiter } - queueLimiter.SetLimitAt(now, rate.Limit(l.schedulingConfig.MaximumPerQueueSchedulingRate)) - if err := sctx.AddQueueSchedulingContext(queue, weight, allocatedByPriorityClass, demand.AggregateByResource(), cappedDemand.AggregateByResource(), queueLimiter); err != nil { return nil, nil, err } diff --git a/internal/scheduler/scheduling_algo_test.go b/internal/scheduler/scheduling_algo_test.go index 7d9afaf944e..2b0625b8707 100644 --- a/internal/scheduler/scheduling_algo_test.go +++ b/internal/scheduler/scheduling_algo_test.go @@ -388,6 +388,57 @@ func TestSchedule(t *testing.T) { queuedJobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass0, 4)), expectedScheduledIndices: []int{0, 1, 2, 3}, }, + "scheduling from paused queue": { + schedulingConfig: testfixtures.TestSchedulingConfig(), + executors: []*schedulerobjects.Executor{ + testfixtures.Test1Node32CoreExecutor("executor1"), + testfixtures.Test1Node32CoreExecutor("executor2"), + }, + queues: []*api.Queue{testfixtures.MakeTestQueueCordoned()}, + queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), + expectedScheduledIndices: []int{}, + }, + "multi-queue scheduling": { + schedulingConfig: testfixtures.TestSchedulingConfig(), + executors: []*schedulerobjects.Executor{ + testfixtures.Test1Node32CoreExecutor("executor1"), + testfixtures.Test1Node32CoreExecutor("executor2"), + }, + queues: []*api.Queue{testfixtures.MakeTestQueue(), testfixtures.MakeTestQueue2()}, + queuedJobs: append( + testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), + testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue2, testfixtures.PriorityClass3, 2)..., + ), + expectedScheduledIndices: []int{0, 1, 10, 11}, + }, + "multi-queue scheduling with paused and non-paused queue": { + schedulingConfig: testfixtures.TestSchedulingConfig(), + executors: []*schedulerobjects.Executor{ + testfixtures.Test1Node32CoreExecutor("executor1"), + testfixtures.Test1Node32CoreExecutor("executor2"), + }, + queues: []*api.Queue{testfixtures.MakeTestQueueCordoned(), testfixtures.MakeTestQueue()}, + queuedJobs: append( + testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), + testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue1, testfixtures.PriorityClass3, 10)..., + ), + expectedScheduledIndices: []int{0, 1, 2, 3}, + }, + "multi-queue scheduling with paused and non-paused queue large": { + schedulingConfig: testfixtures.TestSchedulingConfig(), + executors: []*schedulerobjects.Executor{ + testfixtures.Test1Node32CoreExecutor("executor1"), + testfixtures.Test1Node32CoreExecutor("executor2"), + testfixtures.Test1Node32CoreExecutor("executor3"), + testfixtures.Test1Node32CoreExecutor("executor4"), + }, + queues: []*api.Queue{testfixtures.MakeTestQueueCordoned(), testfixtures.MakeTestQueue()}, + queuedJobs: append( + testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), + testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue1, testfixtures.PriorityClass3, 10)..., + ), + expectedScheduledIndices: []int{0, 1, 2, 3, 4, 5, 6, 7}, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 27244d6db1f..9b59f8297ac 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -500,12 +500,7 @@ func (s *Simulator) handleScheduleEvent(ctx *armadacontext.Context) error { return err } } - constraints := schedulerconstraints.NewSchedulingConstraints( - pool.Name, - totalResources, - s.schedulingConfig, - nil, - ) + constraints := schedulerconstraints.NewSchedulingConstraints(pool.Name, totalResources, s.schedulingConfig, nil, map[string]bool{}) nloatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(s.schedulingConfig.ExperimentalFloatingResources) if err != nil { diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 2fe70108674..11005420266 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -33,6 +33,8 @@ import ( const ( TestJobset = "testJobset" TestQueue = "testQueue" + TestQueue1 = "testQueue1" + TestQueue2 = "testQueue2" TestPool = "testPool" TestHostnameLabel = "kubernetes.io/hostname" ClusterNameLabel = "cluster" @@ -844,6 +846,21 @@ func MakeTestQueue() *api.Queue { } } +func MakeTestQueue2() *api.Queue { + return &api.Queue{ + Name: TestQueue2, + PriorityFactor: 100, + } +} + +func MakeTestQueueCordoned() *api.Queue { + return &api.Queue{ + Name: TestQueue1, + PriorityFactor: 100, + Cordoned: true, + } +} + func TestQueuedJobDbJob() *jobdb.Job { job, _ := JobDb.NewJob( util.NewULID(), diff --git a/pkg/api/api.swagger.go b/pkg/api/api.swagger.go index b7db2275956..b0baf0f1b07 100644 --- a/pkg/api/api.swagger.go +++ b/pkg/api/api.swagger.go @@ -1917,12 +1917,23 @@ func SwaggerJsonTemplate() string { " \"type\": \"object\",\n" + " \"title\": \"swagger:model\",\n" + " \"properties\": {\n" + + " \"cordoned\": {\n" + + " \"description\": \"Determines whether scheduling is enabled for this queue.\",\n" + + " \"type\": \"boolean\"\n" + + " },\n" + " \"groupOwners\": {\n" + " \"type\": \"array\",\n" + " \"items\": {\n" + " \"type\": \"string\"\n" + " }\n" + " },\n" + + " \"labels\": {\n" + + " \"type\": \"array\",\n" + + " \"title\": \"A list of Kubernetes-like key-value labels, e.g. armadaproject.io/priority=critical\",\n" + + " \"items\": {\n" + + " \"type\": \"string\"\n" + + " }\n" + + " },\n" + " \"name\": {\n" + " \"type\": \"string\"\n" + " },\n" + diff --git a/pkg/api/api.swagger.json b/pkg/api/api.swagger.json index f5091b05734..e5ade0ec4b0 100644 --- a/pkg/api/api.swagger.json +++ b/pkg/api/api.swagger.json @@ -1906,12 +1906,23 @@ "type": "object", "title": "swagger:model", "properties": { + "cordoned": { + "description": "Determines whether scheduling is enabled for this queue.", + "type": "boolean" + }, "groupOwners": { "type": "array", "items": { "type": "string" } }, + "labels": { + "type": "array", + "title": "A list of Kubernetes-like key-value labels, e.g. armadaproject.io/priority=critical", + "items": { + "type": "string" + } + }, "name": { "type": "string" }, diff --git a/pkg/api/submit.pb.go b/pkg/api/submit.pb.go index 14f3f690b80..b7bb646e4e0 100644 --- a/pkg/api/submit.pb.go +++ b/pkg/api/submit.pb.go @@ -1167,6 +1167,10 @@ type Queue struct { // If provided for a priority class, global limits for that priority class do not apply to this queue. ResourceLimitsByPriorityClassName map[string]*PriorityClassResourceLimits `protobuf:"bytes,7,rep,name=resource_limits_by_priority_class_name,json=resourceLimitsByPriorityClassName,proto3" json:"resourceLimitsByPriorityClassName,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` Permissions []*Queue_Permissions `protobuf:"bytes,6,rep,name=permissions,proto3" json:"permissions,omitempty"` + // Determines whether scheduling is enabled for this queue. + Cordoned bool `protobuf:"varint,8,opt,name=cordoned,proto3" json:"cordoned,omitempty"` + // A list of Kubernetes-like key-value labels, e.g. armadaproject.io/priority=critical + Labels []string `protobuf:"bytes,9,rep,name=labels,proto3" json:"labels,omitempty"` } func (m *Queue) Reset() { *m = Queue{} } @@ -1252,6 +1256,20 @@ func (m *Queue) GetPermissions() []*Queue_Permissions { return nil } +func (m *Queue) GetCordoned() bool { + if m != nil { + return m.Cordoned + } + return false +} + +func (m *Queue) GetLabels() []string { + if m != nil { + return m.Labels + } + return nil +} + type Queue_Permissions struct { Subjects []*Queue_Permissions_Subject `protobuf:"bytes,1,rep,name=subjects,proto3" json:"subjects,omitempty"` Verbs []string `protobuf:"bytes,2,rep,name=verbs,proto3" json:"verbs,omitempty"` @@ -1591,6 +1609,96 @@ func (m *QueueGetRequest) GetName() string { return "" } +//swagger:model +type QueueCordonRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (m *QueueCordonRequest) Reset() { *m = QueueCordonRequest{} } +func (m *QueueCordonRequest) String() string { return proto.CompactTextString(m) } +func (*QueueCordonRequest) ProtoMessage() {} +func (*QueueCordonRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_e998bacb27df16c1, []int{19} +} +func (m *QueueCordonRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *QueueCordonRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_QueueCordonRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *QueueCordonRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueueCordonRequest.Merge(m, src) +} +func (m *QueueCordonRequest) XXX_Size() int { + return m.Size() +} +func (m *QueueCordonRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueueCordonRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueueCordonRequest proto.InternalMessageInfo + +func (m *QueueCordonRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +//swagger:model +type QueueUncordonRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (m *QueueUncordonRequest) Reset() { *m = QueueUncordonRequest{} } +func (m *QueueUncordonRequest) String() string { return proto.CompactTextString(m) } +func (*QueueUncordonRequest) ProtoMessage() {} +func (*QueueUncordonRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_e998bacb27df16c1, []int{20} +} +func (m *QueueUncordonRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *QueueUncordonRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_QueueUncordonRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *QueueUncordonRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueueUncordonRequest.Merge(m, src) +} +func (m *QueueUncordonRequest) XXX_Size() int { + return m.Size() +} +func (m *QueueUncordonRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueueUncordonRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueueUncordonRequest proto.InternalMessageInfo + +func (m *QueueUncordonRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + //swagger:model type StreamingQueueGetRequest struct { Num uint32 `protobuf:"varint,1,opt,name=num,proto3" json:"num,omitempty"` @@ -1600,7 +1708,7 @@ func (m *StreamingQueueGetRequest) Reset() { *m = StreamingQueueGetReque func (m *StreamingQueueGetRequest) String() string { return proto.CompactTextString(m) } func (*StreamingQueueGetRequest) ProtoMessage() {} func (*StreamingQueueGetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{19} + return fileDescriptor_e998bacb27df16c1, []int{21} } func (m *StreamingQueueGetRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1645,7 +1753,7 @@ func (m *QueueDeleteRequest) Reset() { *m = QueueDeleteRequest{} } func (m *QueueDeleteRequest) String() string { return proto.CompactTextString(m) } func (*QueueDeleteRequest) ProtoMessage() {} func (*QueueDeleteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{20} + return fileDescriptor_e998bacb27df16c1, []int{22} } func (m *QueueDeleteRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1691,7 +1799,7 @@ func (m *JobSetInfo) Reset() { *m = JobSetInfo{} } func (m *JobSetInfo) String() string { return proto.CompactTextString(m) } func (*JobSetInfo) ProtoMessage() {} func (*JobSetInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{21} + return fileDescriptor_e998bacb27df16c1, []int{23} } func (m *JobSetInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1750,7 +1858,7 @@ func (m *QueueUpdateResponse) Reset() { *m = QueueUpdateResponse{} } func (m *QueueUpdateResponse) String() string { return proto.CompactTextString(m) } func (*QueueUpdateResponse) ProtoMessage() {} func (*QueueUpdateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{22} + return fileDescriptor_e998bacb27df16c1, []int{24} } func (m *QueueUpdateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1801,7 +1909,7 @@ func (m *BatchQueueUpdateResponse) Reset() { *m = BatchQueueUpdateRespon func (m *BatchQueueUpdateResponse) String() string { return proto.CompactTextString(m) } func (*BatchQueueUpdateResponse) ProtoMessage() {} func (*BatchQueueUpdateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{23} + return fileDescriptor_e998bacb27df16c1, []int{25} } func (m *BatchQueueUpdateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1846,7 +1954,7 @@ func (m *QueueCreateResponse) Reset() { *m = QueueCreateResponse{} } func (m *QueueCreateResponse) String() string { return proto.CompactTextString(m) } func (*QueueCreateResponse) ProtoMessage() {} func (*QueueCreateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{24} + return fileDescriptor_e998bacb27df16c1, []int{26} } func (m *QueueCreateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1897,7 +2005,7 @@ func (m *BatchQueueCreateResponse) Reset() { *m = BatchQueueCreateRespon func (m *BatchQueueCreateResponse) String() string { return proto.CompactTextString(m) } func (*BatchQueueCreateResponse) ProtoMessage() {} func (*BatchQueueCreateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{25} + return fileDescriptor_e998bacb27df16c1, []int{27} } func (m *BatchQueueCreateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1941,7 +2049,7 @@ func (m *EndMarker) Reset() { *m = EndMarker{} } func (m *EndMarker) String() string { return proto.CompactTextString(m) } func (*EndMarker) ProtoMessage() {} func (*EndMarker) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{26} + return fileDescriptor_e998bacb27df16c1, []int{28} } func (m *EndMarker) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1981,7 +2089,7 @@ func (m *StreamingQueueMessage) Reset() { *m = StreamingQueueMessage{} } func (m *StreamingQueueMessage) String() string { return proto.CompactTextString(m) } func (*StreamingQueueMessage) ProtoMessage() {} func (*StreamingQueueMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_e998bacb27df16c1, []int{27} + return fileDescriptor_e998bacb27df16c1, []int{29} } func (m *StreamingQueueMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2093,6 +2201,8 @@ func init() { proto.RegisterType((*QueueList)(nil), "api.QueueList") proto.RegisterType((*CancellationResult)(nil), "api.CancellationResult") proto.RegisterType((*QueueGetRequest)(nil), "api.QueueGetRequest") + proto.RegisterType((*QueueCordonRequest)(nil), "api.QueueCordonRequest") + proto.RegisterType((*QueueUncordonRequest)(nil), "api.QueueUncordonRequest") proto.RegisterType((*StreamingQueueGetRequest)(nil), "api.StreamingQueueGetRequest") proto.RegisterType((*QueueDeleteRequest)(nil), "api.QueueDeleteRequest") proto.RegisterType((*JobSetInfo)(nil), "api.JobSetInfo") @@ -2107,192 +2217,197 @@ func init() { func init() { proto.RegisterFile("pkg/api/submit.proto", fileDescriptor_e998bacb27df16c1) } var fileDescriptor_e998bacb27df16c1 = []byte{ - // 2952 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x1b, 0xc7, - 0x15, 0xd7, 0x8a, 0xfa, 0xe2, 0xa3, 0x3e, 0xa8, 0xd1, 0x87, 0x57, 0xb4, 0x23, 0xca, 0x9b, 0xc4, - 0xa1, 0x15, 0x97, 0x8c, 0x95, 0x06, 0xb5, 0xdd, 0x14, 0xae, 0x48, 0xd1, 0xb6, 0x64, 0x9b, 0x56, - 0x28, 0x2b, 0x1f, 0x45, 0x51, 0x76, 0xc9, 0x1d, 0x51, 0x2b, 0x91, 0xbb, 0xcc, 0xee, 0x52, 0xae, - 0x5a, 0xe4, 0x52, 0x14, 0xe8, 0xa1, 0x97, 0xa0, 0x3d, 0x16, 0x68, 0x0a, 0xb4, 0xa7, 0xf4, 0x5c, - 0xa0, 0x28, 0xfa, 0x07, 0xf4, 0x18, 0xa0, 0x97, 0x16, 0x05, 0x88, 0x22, 0x6e, 0x1b, 0x80, 0xb7, - 0xde, 0x7b, 0x28, 0xe6, 0xcd, 0x2e, 0x77, 0x96, 0x1f, 0x22, 0x69, 0x5b, 0xc9, 0xa5, 0x37, 0xed, - 0x6f, 0xde, 0xf7, 0xbc, 0x79, 0xf3, 0xde, 0x50, 0xb0, 0x58, 0x3b, 0x2e, 0xa7, 0xd4, 0x9a, 0x9e, - 0xb2, 0xeb, 0xc5, 0xaa, 0xee, 0x24, 0x6b, 0x96, 0xe9, 0x98, 0x24, 0xa4, 0xd6, 0xf4, 0xd8, 0xc5, - 0xb2, 0x69, 0x96, 0x2b, 0x34, 0x85, 0x50, 0xb1, 0x7e, 0x90, 0xa2, 0xd5, 0x9a, 0x73, 0xca, 0x29, - 0x62, 0xf1, 0xf6, 0x45, 0x47, 0xaf, 0x52, 0xdb, 0x51, 0xab, 0x35, 0x97, 0x40, 0x39, 0xbe, 0x61, - 0x27, 0x75, 0x13, 0x65, 0x97, 0x4c, 0x8b, 0xa6, 0x4e, 0xae, 0xa7, 0xca, 0xd4, 0xa0, 0x96, 0xea, - 0x50, 0xcd, 0xa5, 0x49, 0x08, 0x34, 0x06, 0x75, 0x9e, 0x98, 0xd6, 0xb1, 0x6e, 0x94, 0xbb, 0x51, - 0x5e, 0x72, 0xd5, 0x31, 0x4a, 0xd5, 0x30, 0x4c, 0x47, 0x75, 0x74, 0xd3, 0xb0, 0xdd, 0xd5, 0x96, - 0x13, 0x87, 0x54, 0xad, 0x38, 0x87, 0x1c, 0x55, 0x3e, 0x09, 0xc3, 0xe2, 0x8e, 0x59, 0xdc, 0x43, - 0xc7, 0xf2, 0xf4, 0xc3, 0x3a, 0xb5, 0x9d, 0x6d, 0x87, 0x56, 0xc9, 0x06, 0x4c, 0xd5, 0x2c, 0xdd, - 0xb4, 0x74, 0xe7, 0x54, 0x96, 0xd6, 0xa4, 0x84, 0x94, 0x5e, 0x6e, 0x36, 0xe2, 0xc4, 0xc3, 0xae, - 0x99, 0x55, 0xdd, 0x41, 0x5f, 0xf3, 0x2d, 0x3a, 0xf2, 0x16, 0x84, 0x0d, 0xb5, 0x4a, 0xed, 0x9a, - 0x5a, 0xa2, 0x72, 0x68, 0x4d, 0x4a, 0x84, 0xd3, 0x17, 0x9a, 0x8d, 0xf8, 0x42, 0x0b, 0x14, 0xb8, - 0x7c, 0x4a, 0xf2, 0x26, 0x84, 0x4b, 0x15, 0x9d, 0x1a, 0x4e, 0x41, 0xd7, 0xe4, 0x29, 0x64, 0x43, - 0x5d, 0x1c, 0xdc, 0xd6, 0x44, 0x5d, 0x1e, 0x46, 0xf6, 0x60, 0xa2, 0xa2, 0x16, 0x69, 0xc5, 0x96, - 0xc7, 0xd6, 0x42, 0x89, 0xc8, 0xc6, 0xab, 0x49, 0xb5, 0xa6, 0x27, 0xbb, 0xb9, 0x92, 0x7c, 0x80, - 0x74, 0x59, 0xc3, 0xb1, 0x4e, 0xd3, 0x8b, 0xcd, 0x46, 0x3c, 0xca, 0x19, 0x05, 0xb1, 0xae, 0x28, - 0x52, 0x86, 0x88, 0x10, 0x38, 0x79, 0x1c, 0x25, 0xaf, 0xf7, 0x96, 0xbc, 0xe9, 0x13, 0x73, 0xf1, - 0x2b, 0xcd, 0x46, 0x7c, 0x49, 0x10, 0x21, 0xe8, 0x10, 0x25, 0x93, 0x9f, 0x4a, 0xb0, 0x68, 0xd1, - 0x0f, 0xeb, 0xba, 0x45, 0xb5, 0x82, 0x61, 0x6a, 0xb4, 0xe0, 0x3a, 0x33, 0x81, 0x2a, 0xaf, 0xf7, - 0x56, 0x99, 0x77, 0xb9, 0x72, 0xa6, 0x46, 0x45, 0xc7, 0x94, 0x66, 0x23, 0x7e, 0xc9, 0xea, 0x58, - 0xf4, 0x0d, 0x90, 0xa5, 0x3c, 0xe9, 0x5c, 0x27, 0x8f, 0x60, 0xaa, 0x66, 0x6a, 0x05, 0xbb, 0x46, - 0x4b, 0xf2, 0xe8, 0x9a, 0x94, 0x88, 0x6c, 0x5c, 0x4c, 0xf2, 0x8c, 0x43, 0x1b, 0x58, 0x56, 0x26, - 0x4f, 0xae, 0x27, 0x77, 0x4d, 0x6d, 0xaf, 0x46, 0x4b, 0xb8, 0x9f, 0xf3, 0x35, 0xfe, 0x11, 0x90, - 0x3d, 0xe9, 0x82, 0x64, 0x17, 0xc2, 0x9e, 0x40, 0x5b, 0x9e, 0x44, 0x77, 0xce, 0x94, 0xc8, 0xd3, - 0x8a, 0x7f, 0xd8, 0x81, 0xb4, 0x72, 0x31, 0x92, 0x81, 0x49, 0xdd, 0x28, 0x5b, 0xd4, 0xb6, 0xe5, - 0x30, 0xca, 0x23, 0x28, 0x68, 0x9b, 0x63, 0x19, 0xd3, 0x38, 0xd0, 0xcb, 0xe9, 0x25, 0x66, 0x98, - 0x4b, 0x26, 0x48, 0xf1, 0x38, 0xc9, 0x1d, 0x98, 0xb2, 0xa9, 0x75, 0xa2, 0x97, 0xa8, 0x2d, 0x83, - 0x20, 0x65, 0x8f, 0x83, 0xae, 0x14, 0x34, 0xc6, 0xa3, 0x13, 0x8d, 0xf1, 0x30, 0x96, 0xe3, 0x76, - 0xe9, 0x90, 0x6a, 0xf5, 0x0a, 0xb5, 0xe4, 0x88, 0x9f, 0xe3, 0x2d, 0x50, 0xcc, 0xf1, 0x16, 0x18, - 0x53, 0x21, 0x22, 0xec, 0x16, 0x79, 0x19, 0x42, 0xc7, 0x94, 0x1f, 0xac, 0x70, 0x7a, 0xbe, 0xd9, - 0x88, 0xcf, 0x1c, 0x53, 0xf1, 0x4c, 0xb1, 0x55, 0x72, 0x15, 0xc6, 0x4f, 0xd4, 0x4a, 0x9d, 0xe2, - 0xbe, 0x84, 0xd3, 0x0b, 0xcd, 0x46, 0x7c, 0x0e, 0x01, 0x81, 0x90, 0x53, 0xdc, 0x1a, 0xbd, 0x21, - 0xc5, 0x0e, 0x20, 0xda, 0x9e, 0x8f, 0xe7, 0xa2, 0xa7, 0x0a, 0x17, 0x7a, 0x24, 0xe1, 0x79, 0xa8, - 0x53, 0xfe, 0x13, 0x82, 0x99, 0xc0, 0x56, 0x93, 0x5b, 0x30, 0xe6, 0x9c, 0xd6, 0x28, 0xaa, 0x99, - 0xdd, 0x88, 0x8a, 0xc9, 0xf0, 0xf8, 0xb4, 0x46, 0xf1, 0x8c, 0xcf, 0x32, 0x8a, 0x40, 0x82, 0x22, - 0x0f, 0x53, 0x5e, 0x33, 0x2d, 0xc7, 0x96, 0x47, 0xd7, 0x42, 0x89, 0x19, 0xae, 0x1c, 0x01, 0x51, - 0x39, 0x02, 0xe4, 0xfb, 0xc1, 0x62, 0x10, 0xc2, 0xa4, 0x79, 0xb9, 0x33, 0xf5, 0x9e, 0xbd, 0x0a, - 0xdc, 0x84, 0x88, 0x53, 0xb1, 0x0b, 0xd4, 0x50, 0x8b, 0x15, 0xaa, 0xc9, 0x63, 0x6b, 0x52, 0x62, - 0x2a, 0x2d, 0x37, 0x1b, 0xf1, 0x45, 0x87, 0x45, 0x14, 0x51, 0x81, 0x17, 0x7c, 0x14, 0x6b, 0x26, - 0xb5, 0x9c, 0x02, 0xab, 0xa2, 0xf2, 0xb8, 0x50, 0x33, 0xa9, 0xe5, 0xe4, 0xd4, 0x2a, 0x0d, 0xd4, - 0x4c, 0x17, 0x23, 0xb7, 0x61, 0xa6, 0x6e, 0xd3, 0x42, 0xa9, 0x52, 0xb7, 0x1d, 0x6a, 0x6d, 0xef, - 0xca, 0x13, 0xa8, 0x31, 0xd6, 0x6c, 0xc4, 0x97, 0xeb, 0x36, 0xcd, 0x78, 0xb8, 0xc0, 0x3c, 0x2d, - 0xe2, 0x5f, 0x56, 0x8a, 0x29, 0x0e, 0xcc, 0x04, 0xce, 0x25, 0xb9, 0xd1, 0x65, 0xcb, 0x5d, 0x0a, - 0xdc, 0x72, 0xd2, 0xb9, 0xe5, 0x43, 0x6f, 0xb8, 0xf2, 0x37, 0x09, 0xa2, 0xed, 0x35, 0x97, 0xf1, - 0x7f, 0x58, 0xa7, 0x75, 0xea, 0x3a, 0x88, 0xfc, 0x08, 0x88, 0xfc, 0x08, 0x90, 0xaf, 0x03, 0x1c, - 0x99, 0xc5, 0x82, 0x4d, 0xf1, 0x22, 0x1b, 0xf5, 0x37, 0xe5, 0xc8, 0x2c, 0xee, 0xd1, 0xb6, 0x8b, - 0xcc, 0xc3, 0x88, 0x06, 0xf3, 0x8c, 0xcb, 0xe2, 0xfa, 0x0a, 0x8c, 0xc0, 0x4b, 0xb6, 0x95, 0x9e, - 0xd7, 0x40, 0xfa, 0xa5, 0x66, 0x23, 0xbe, 0x72, 0x64, 0x16, 0x05, 0x4c, 0xf4, 0x68, 0xae, 0x6d, - 0x49, 0xf9, 0x8d, 0x04, 0xf3, 0x3b, 0x66, 0x71, 0xd7, 0xa2, 0x8c, 0xe0, 0x4b, 0x73, 0xee, 0x6b, - 0x30, 0xc9, 0xb8, 0x74, 0x8d, 0xbb, 0x14, 0xe6, 0xf7, 0xef, 0x91, 0x59, 0xdc, 0xd6, 0x02, 0xf7, - 0x2f, 0x47, 0x94, 0xff, 0xf2, 0x1d, 0xc8, 0xa8, 0x46, 0x89, 0x56, 0x3c, 0x23, 0xd7, 0x61, 0x82, - 0xcb, 0x10, 0xad, 0x44, 0x06, 0xd1, 0x4a, 0x04, 0x9e, 0xd1, 0xca, 0x56, 0x18, 0x42, 0x7d, 0xc3, - 0x20, 0x38, 0x34, 0xd6, 0xdf, 0x21, 0x72, 0x0d, 0x26, 0x2c, 0xaa, 0xda, 0xa6, 0xe1, 0x9e, 0x51, - 0xa4, 0xe6, 0x88, 0x48, 0xcd, 0x11, 0xe5, 0x5f, 0x12, 0x2c, 0xec, 0xa0, 0x51, 0xc1, 0x08, 0x04, - 0xbd, 0x92, 0x86, 0xf5, 0x6a, 0xb4, 0xaf, 0x57, 0xb7, 0x61, 0xe2, 0x40, 0xaf, 0x38, 0xd4, 0xc2, - 0x08, 0x44, 0x36, 0xe6, 0x5b, 0x89, 0x47, 0x9d, 0x3b, 0xb8, 0xc0, 0x2d, 0xe7, 0x44, 0xa2, 0xe5, - 0x1c, 0x11, 0xfc, 0x1c, 0x1b, 0xc0, 0xcf, 0xfb, 0x30, 0x2d, 0xca, 0x26, 0xdf, 0x84, 0x09, 0xdb, - 0x51, 0x1d, 0x6a, 0xcb, 0xd2, 0x5a, 0x28, 0x31, 0xbb, 0x31, 0xd3, 0x52, 0xcf, 0x50, 0x2e, 0x8c, - 0x13, 0x88, 0xc2, 0x38, 0xa2, 0x7c, 0x31, 0x07, 0xa1, 0x1d, 0xb3, 0x48, 0xd6, 0x60, 0xb4, 0x15, - 0x9c, 0x68, 0xb3, 0x11, 0x9f, 0xd6, 0xc5, 0xb0, 0x8c, 0xea, 0x5a, 0xb0, 0xcf, 0x9c, 0x19, 0xb0, - 0xcf, 0x3c, 0xf7, 0x8c, 0x0a, 0x34, 0xcd, 0x93, 0x03, 0x37, 0xcd, 0xe9, 0x56, 0xff, 0xcb, 0x7b, - 0xa2, 0x45, 0x2f, 0x66, 0x43, 0xb4, 0xbb, 0xef, 0x06, 0x6f, 0x38, 0x08, 0x16, 0x9d, 0x67, 0xbf, - 0xd7, 0x4e, 0x7a, 0x34, 0xb7, 0x11, 0x54, 0xb0, 0xd6, 0x52, 0xf0, 0xa2, 0x7b, 0xd9, 0xab, 0x30, - 0x6e, 0x3e, 0x31, 0xa8, 0xe5, 0x0e, 0x11, 0x18, 0x75, 0x04, 0xc4, 0xa8, 0x23, 0x40, 0x28, 0x5c, - 0xc4, 0xf0, 0x17, 0xf0, 0xd3, 0x3e, 0xd4, 0x6b, 0x85, 0xba, 0x4d, 0xad, 0x42, 0xd9, 0x32, 0xeb, - 0x35, 0x5b, 0x9e, 0xc3, 0xb3, 0x7d, 0xa5, 0xd9, 0x88, 0x2b, 0x48, 0xf6, 0xc8, 0xa3, 0xda, 0xb7, - 0xa9, 0x75, 0x17, 0x69, 0x04, 0x99, 0x72, 0x2f, 0x1a, 0xf2, 0x13, 0x09, 0xae, 0x94, 0xcc, 0x6a, - 0x8d, 0x75, 0x0b, 0x54, 0x2b, 0x9c, 0xa5, 0x72, 0x61, 0x4d, 0x4a, 0x4c, 0xa7, 0xdf, 0x68, 0x36, - 0xe2, 0xd7, 0x7c, 0x8e, 0x77, 0xfa, 0x2b, 0x57, 0xfa, 0x53, 0x07, 0x86, 0xb9, 0xb1, 0x01, 0x87, - 0x39, 0x71, 0x30, 0x18, 0x7f, 0xe1, 0x83, 0xc1, 0xf4, 0x8b, 0x18, 0x0c, 0x7e, 0x2d, 0xc1, 0x9a, - 0xdb, 0x62, 0xeb, 0x46, 0xb9, 0x60, 0x51, 0xdb, 0xac, 0x5b, 0x25, 0x5a, 0x70, 0x53, 0xa3, 0x4a, - 0x0d, 0xc7, 0x96, 0x97, 0xd0, 0xf6, 0x44, 0x37, 0x4d, 0x79, 0x97, 0x21, 0x2f, 0xd0, 0xa7, 0xaf, - 0x35, 0x1b, 0xf1, 0x84, 0x2f, 0xb5, 0x1b, 0x8d, 0x60, 0xcc, 0xea, 0xd9, 0x94, 0xe4, 0x3e, 0x4c, - 0x96, 0x2c, 0xca, 0x86, 0x74, 0x6c, 0xb6, 0x22, 0x1b, 0xb1, 0x24, 0x9f, 0xd2, 0x93, 0xde, 0xa3, - 0x40, 0xf2, 0xb1, 0xf7, 0x28, 0xc0, 0x67, 0x18, 0x97, 0x5c, 0x9c, 0x61, 0x5c, 0x48, 0x1c, 0x84, - 0x66, 0x5f, 0xc8, 0x20, 0x14, 0x7d, 0x8e, 0x41, 0xe8, 0xbb, 0x10, 0x39, 0xbe, 0x61, 0x17, 0x3c, - 0x83, 0xe6, 0x51, 0xd4, 0x65, 0x31, 0xcc, 0xfe, 0x6b, 0x05, 0x0b, 0xb6, 0x6b, 0x25, 0xef, 0x6f, - 0x8f, 0x6f, 0xd8, 0xdb, 0x1d, 0x26, 0x82, 0x8f, 0xb2, 0xd2, 0xc4, 0xa4, 0xbb, 0xda, 0x64, 0xd2, - 0x3b, 0x5d, 0x5c, 0xbb, 0x5b, 0x72, 0xdd, 0xef, 0x36, 0xb9, 0x2e, 0x1a, 0x1c, 0xdf, 0x16, 0x07, - 0x1d, 0xdf, 0xc8, 0x36, 0xcc, 0xf3, 0xb3, 0xeb, 0x38, 0x95, 0x82, 0x4d, 0x4b, 0xa6, 0xa1, 0xd9, - 0xf2, 0xf2, 0x9a, 0x94, 0x08, 0xf1, 0x4e, 0x0c, 0x17, 0x1f, 0x3b, 0x95, 0x3d, 0xbe, 0x24, 0x76, - 0x62, 0x6d, 0x4b, 0xff, 0x9f, 0x04, 0x9f, 0x79, 0x2a, 0xf8, 0xb7, 0x04, 0xcb, 0x3b, 0xac, 0xaf, - 0x75, 0x6b, 0x94, 0xfe, 0x43, 0xea, 0x75, 0x48, 0x42, 0x5b, 0x26, 0x0d, 0xd0, 0x96, 0x9d, 0xfb, - 0xa5, 0xfe, 0x36, 0x4c, 0x1b, 0xf4, 0x49, 0xa1, 0xad, 0xe8, 0xe2, 0xfd, 0x69, 0xd0, 0x27, 0xbb, - 0x9d, 0x75, 0x37, 0x22, 0xc0, 0xca, 0xef, 0x46, 0xe1, 0x42, 0x87, 0xa3, 0x76, 0xcd, 0x34, 0x6c, - 0x4a, 0x7e, 0x29, 0x81, 0x6c, 0xf9, 0x0b, 0xb8, 0xc5, 0xac, 0xf2, 0xd5, 0x2b, 0x0e, 0xf7, 0x3d, - 0xb2, 0x71, 0xd3, 0xbb, 0x60, 0xbb, 0x09, 0x48, 0xe6, 0xdb, 0x98, 0xf3, 0x9c, 0x97, 0xdf, 0xbc, - 0xaf, 0x36, 0x1b, 0xf1, 0xcb, 0x56, 0x77, 0x0a, 0xc1, 0xda, 0x0b, 0x3d, 0x48, 0x62, 0x16, 0x5c, - 0x3a, 0x4b, 0xfe, 0xb9, 0xa4, 0x85, 0x01, 0x4b, 0xc2, 0x88, 0xc4, 0xbd, 0xc4, 0x27, 0xcc, 0x61, - 0x06, 0x87, 0xab, 0x30, 0x4e, 0x2d, 0xcb, 0xb4, 0x44, 0x9d, 0x08, 0x88, 0xa4, 0x08, 0x28, 0x1f, - 0xe1, 0x24, 0x15, 0xd4, 0x47, 0x0e, 0x81, 0xf0, 0x29, 0x8e, 0x7f, 0xbb, 0x63, 0x1c, 0xdf, 0x8f, - 0x58, 0xfb, 0x18, 0xe7, 0xdb, 0x98, 0x5e, 0x6d, 0x36, 0xe2, 0x31, 0x1c, 0xd6, 0x7c, 0x50, 0x8c, - 0x74, 0xb4, 0x7d, 0x4d, 0xf9, 0xfb, 0x14, 0x8c, 0xe3, 0x45, 0x4f, 0xae, 0xc0, 0x18, 0x8e, 0xff, - 0xdc, 0x3b, 0x1c, 0x81, 0x8d, 0xe0, 0xe8, 0x8f, 0xeb, 0x24, 0x0b, 0x73, 0x5e, 0x22, 0x16, 0x0e, - 0xd4, 0x92, 0xe3, 0x7a, 0x29, 0xa5, 0x2f, 0x35, 0x1b, 0x71, 0xd9, 0x5b, 0xba, 0x83, 0x2b, 0x02, - 0xf3, 0x6c, 0x70, 0x85, 0xdc, 0x84, 0x08, 0xf6, 0x2b, 0xbc, 0x7d, 0x71, 0xe7, 0x39, 0xac, 0xba, - 0x0c, 0xe6, 0x6d, 0x87, 0x58, 0x75, 0x7d, 0x94, 0x1d, 0x07, 0xec, 0x72, 0x3c, 0x5e, 0x3e, 0x3a, - 0xe1, 0x71, 0x40, 0xbc, 0x83, 0x39, 0x22, 0xc0, 0xa4, 0x0c, 0x73, 0xad, 0xab, 0xbd, 0xa2, 0x57, - 0x75, 0xc7, 0x7b, 0x99, 0x5d, 0xc5, 0xc0, 0x62, 0x30, 0x5a, 0x77, 0xf9, 0x03, 0x24, 0xe0, 0xd9, - 0xcc, 0x82, 0x2b, 0x5b, 0x81, 0x85, 0x40, 0x6b, 0x32, 0x1b, 0x5c, 0x23, 0xbf, 0x97, 0xe0, 0x4a, - 0x9b, 0xa6, 0x42, 0xf1, 0xb4, 0x75, 0x8a, 0x0b, 0xa5, 0x8a, 0x6a, 0xdb, 0xfc, 0xc9, 0x65, 0x52, - 0x78, 0xa7, 0xed, 0x66, 0x40, 0xfa, 0xd4, 0x3b, 0xcd, 0x19, 0xc6, 0x94, 0x53, 0xab, 0x94, 0xdb, - 0x94, 0x6a, 0x36, 0xe2, 0xaf, 0x5b, 0xfd, 0x68, 0x85, 0x50, 0x5c, 0xee, 0x4b, 0x4c, 0xf6, 0x20, - 0x52, 0xa3, 0x56, 0x55, 0xb7, 0x6d, 0xec, 0xe3, 0xf9, 0x1b, 0xf2, 0xb2, 0x60, 0xdb, 0xae, 0xbf, - 0xca, 0xa3, 0x2e, 0x90, 0x8b, 0x51, 0x17, 0xe0, 0xd8, 0x17, 0x12, 0x44, 0x04, 0x3e, 0x92, 0x87, - 0x29, 0xbb, 0x5e, 0x3c, 0xa2, 0xa5, 0x56, 0x9d, 0x59, 0xed, 0xae, 0x21, 0xb9, 0xc7, 0xc9, 0xdc, - 0x1e, 0xc2, 0xe5, 0x09, 0xf4, 0x10, 0x2e, 0x86, 0x27, 0x9d, 0x5a, 0x45, 0xfe, 0x38, 0xe3, 0x9d, - 0x74, 0x06, 0x04, 0x4e, 0x3a, 0x03, 0x62, 0x1f, 0xc0, 0xa4, 0x2b, 0x97, 0xe5, 0xfd, 0xb1, 0x6e, - 0x68, 0x62, 0xde, 0xb3, 0x6f, 0x31, 0xef, 0xd9, 0x77, 0xeb, 0x7c, 0x8c, 0x9e, 0x7d, 0x3e, 0x62, - 0x3a, 0x2c, 0x74, 0xc9, 0x9e, 0x67, 0xa8, 0x55, 0x52, 0xdf, 0x1b, 0xf3, 0x57, 0x12, 0x5c, 0x19, - 0x2c, 0x51, 0x06, 0x53, 0x7f, 0x5f, 0x54, 0xef, 0x8d, 0x56, 0x01, 0x81, 0x6d, 0xda, 0xfa, 0x15, - 0xd3, 0x9f, 0x8f, 0xc3, 0xc5, 0x33, 0xf8, 0x59, 0xcb, 0xbd, 0x52, 0x55, 0x7f, 0xa0, 0x57, 0xeb, - 0x55, 0xbf, 0xdf, 0x3e, 0xb0, 0xd4, 0x12, 0x2b, 0xf5, 0x6e, 0x5e, 0x7c, 0xab, 0x9f, 0x15, 0xc9, - 0x87, 0x5c, 0x82, 0x87, 0xde, 0x71, 0xf9, 0x85, 0x3b, 0xa8, 0xda, 0x9d, 0x42, 0xbc, 0x83, 0x7a, - 0x90, 0x90, 0x3f, 0x4a, 0x70, 0xb9, 0xa7, 0x89, 0x78, 0x9e, 0x4d, 0xb3, 0x82, 0x19, 0x17, 0xd9, - 0xc8, 0x3c, 0xab, 0xa9, 0xe9, 0xd3, 0x5d, 0xd3, 0xac, 0x70, 0x83, 0x5f, 0x6f, 0x36, 0xe2, 0xaf, - 0x55, 0xcf, 0xa2, 0x13, 0xcc, 0x7e, 0xe9, 0x4c, 0x42, 0x76, 0x81, 0x9e, 0x15, 0x9c, 0xf3, 0x4a, - 0x4a, 0xa5, 0xbf, 0x9b, 0x83, 0xa9, 0x7e, 0x14, 0x4c, 0xc8, 0x57, 0x3a, 0xe3, 0xcb, 0x04, 0x0e, - 0x99, 0x94, 0x7f, 0x1a, 0x85, 0x78, 0x1f, 0x19, 0xe4, 0xb7, 0x03, 0x24, 0xe6, 0xe6, 0x20, 0xd6, - 0x9c, 0x6b, 0x72, 0x7e, 0x15, 0xfb, 0xab, 0x64, 0x21, 0x8c, 0x45, 0xfa, 0x81, 0x6e, 0x3b, 0xe4, - 0x06, 0x4c, 0x60, 0x8b, 0xea, 0x15, 0x71, 0xf0, 0x8b, 0x38, 0x6f, 0x9a, 0xf9, 0xaa, 0xd8, 0x34, - 0x73, 0x44, 0xd9, 0x07, 0xc2, 0x9f, 0x25, 0x2b, 0x42, 0x5f, 0x47, 0x6e, 0xc3, 0x4c, 0x89, 0xa3, - 0x54, 0x13, 0xfa, 0x6f, 0xfc, 0x4d, 0xa1, 0xb5, 0x10, 0xec, 0xc2, 0xa7, 0x45, 0x5c, 0xb9, 0x09, - 0x73, 0xa8, 0xfd, 0x2e, 0x6d, 0x3d, 0x4b, 0x0f, 0xd8, 0xd8, 0x28, 0xb7, 0x41, 0xde, 0x73, 0x2c, - 0xaa, 0x56, 0x75, 0xa3, 0xdc, 0x2e, 0xe3, 0x65, 0x08, 0x19, 0xf5, 0x2a, 0x8a, 0x98, 0xe1, 0x81, - 0x34, 0xea, 0x55, 0x31, 0x90, 0x46, 0xbd, 0xaa, 0xbc, 0x0d, 0x04, 0xf9, 0xb6, 0x68, 0x85, 0x3a, - 0x74, 0x58, 0xf5, 0x9f, 0x4a, 0x00, 0xfc, 0x1d, 0x73, 0xdb, 0x38, 0x30, 0x07, 0x6e, 0xc7, 0x6e, - 0x42, 0x04, 0x23, 0xaa, 0x15, 0x8e, 0x4c, 0xbc, 0xfa, 0xa4, 0xc4, 0x38, 0xef, 0xa3, 0x38, 0xbc, - 0x63, 0x06, 0xee, 0x3f, 0xf0, 0x51, 0xc6, 0x5a, 0xa1, 0xaa, 0xed, 0xb1, 0x86, 0x7c, 0x56, 0x0e, - 0xb7, 0xb3, 0xfa, 0xa8, 0xf2, 0x04, 0x16, 0xd0, 0xd5, 0xfd, 0x9a, 0xa6, 0x3a, 0xfe, 0x38, 0xf1, - 0x96, 0xf8, 0x0b, 0x40, 0x30, 0x1b, 0xce, 0x9a, 0x6f, 0x86, 0x68, 0x97, 0xeb, 0x20, 0xa7, 0x55, - 0xa7, 0x74, 0xd8, 0x4d, 0xfb, 0x07, 0x30, 0x73, 0xa0, 0xea, 0x15, 0xef, 0x65, 0xcc, 0xcb, 0x49, - 0xd9, 0xb7, 0x22, 0xc8, 0xc0, 0xd3, 0x8a, 0xb3, 0xbc, 0xd3, 0x9e, 0xa7, 0xd3, 0x22, 0xde, 0xf2, - 0x37, 0x83, 0x6f, 0x27, 0x5f, 0x95, 0xbf, 0x6d, 0xda, 0xfb, 0xfb, 0x1b, 0x64, 0x18, 0xc2, 0xdf, - 0x08, 0x84, 0xb3, 0x86, 0xf6, 0x50, 0xb5, 0x8e, 0xa9, 0xa5, 0x7c, 0x2c, 0xc1, 0x52, 0xf0, 0x64, - 0x3c, 0xa4, 0xb6, 0xad, 0x96, 0x29, 0xf9, 0xc6, 0x70, 0xfe, 0xdf, 0x1b, 0xf1, 0x9f, 0xa9, 0x43, - 0xd4, 0xd0, 0xdc, 0xb2, 0x3e, 0x8b, 0x6c, 0x2d, 0x7d, 0xfc, 0x7c, 0x51, 0xb1, 0x05, 0xbb, 0x37, - 0x92, 0x67, 0xf4, 0xe9, 0x49, 0x18, 0xa7, 0x27, 0xd4, 0x70, 0xd6, 0x63, 0x10, 0x11, 0x7e, 0xa3, - 0x25, 0x11, 0x98, 0x74, 0x3f, 0xa3, 0x23, 0xeb, 0x57, 0x21, 0x22, 0xfc, 0x98, 0x47, 0xa6, 0x61, - 0x2a, 0x67, 0x6a, 0x74, 0xd7, 0xb4, 0x9c, 0xe8, 0x08, 0xfb, 0xba, 0x47, 0x55, 0xad, 0xc2, 0x48, - 0xa5, 0xf5, 0x4f, 0x24, 0x98, 0xf2, 0x7e, 0x18, 0x20, 0x00, 0x13, 0xef, 0xec, 0x67, 0xf7, 0xb3, - 0x5b, 0xd1, 0x11, 0x26, 0x70, 0x37, 0x9b, 0xdb, 0xda, 0xce, 0xdd, 0x8d, 0x4a, 0xec, 0x23, 0xbf, - 0x9f, 0xcb, 0xb1, 0x8f, 0x51, 0x32, 0x03, 0xe1, 0xbd, 0xfd, 0x4c, 0x26, 0x9b, 0xdd, 0xca, 0x6e, - 0x45, 0x43, 0x8c, 0xe9, 0xce, 0xe6, 0xf6, 0x83, 0xec, 0x56, 0x74, 0x8c, 0xd1, 0xed, 0xe7, 0xee, - 0xe7, 0x1e, 0xbd, 0x97, 0x8b, 0x8e, 0x73, 0xba, 0xf4, 0xc3, 0xed, 0xc7, 0x8f, 0xb3, 0x5b, 0xd1, - 0x09, 0x46, 0xf7, 0x20, 0xbb, 0xb9, 0x97, 0xdd, 0x8a, 0x4e, 0xb2, 0xa5, 0xdd, 0x7c, 0x36, 0xfb, - 0x70, 0x97, 0x2d, 0x4d, 0xb1, 0xcf, 0xcc, 0x66, 0x2e, 0x93, 0x7d, 0xc0, 0xa4, 0x84, 0x99, 0x85, - 0xf9, 0xec, 0x4e, 0x36, 0xc3, 0x16, 0x61, 0xe3, 0x0f, 0x21, 0x98, 0xc6, 0x80, 0x7a, 0x4f, 0x4e, - 0x6f, 0x42, 0x84, 0xef, 0x2a, 0x9f, 0xda, 0x84, 0x90, 0xc7, 0x96, 0x3b, 0x1e, 0x03, 0xb3, 0x2c, - 0x78, 0xca, 0x08, 0xb9, 0x0d, 0xd3, 0x02, 0x93, 0x4d, 0x66, 0x7d, 0x2e, 0x56, 0xc6, 0x63, 0x2f, - 0xe1, 0x77, 0xaf, 0x44, 0x53, 0x46, 0x98, 0x56, 0x7e, 0x76, 0x86, 0xd4, 0x2a, 0x30, 0xf5, 0xd7, - 0x1a, 0x3c, 0x9d, 0xca, 0x08, 0xf9, 0x36, 0x44, 0x78, 0x2d, 0xe5, 0x5a, 0x2f, 0xf8, 0xfc, 0x81, - 0x12, 0x7b, 0x86, 0x09, 0x49, 0x98, 0xba, 0x4b, 0x1d, 0xce, 0xbe, 0xe8, 0xb3, 0xfb, 0x95, 0x3d, - 0x26, 0xb8, 0xa2, 0x8c, 0x90, 0x1d, 0x08, 0x7b, 0xf4, 0x36, 0xe1, 0xf6, 0xf5, 0xba, 0x13, 0x62, - 0xb1, 0x2e, 0xcb, 0xee, 0xc1, 0x50, 0x46, 0xde, 0x90, 0x36, 0x7e, 0x16, 0x86, 0x09, 0x3e, 0xa3, - 0x93, 0x77, 0x01, 0xf8, 0x5f, 0x58, 0x77, 0x97, 0xba, 0xfe, 0x10, 0x1b, 0x5b, 0xee, 0x3e, 0xd8, - 0x2b, 0x2b, 0x3f, 0xfe, 0xcb, 0x3f, 0x7f, 0x31, 0xba, 0xa0, 0xcc, 0xa6, 0x4e, 0xae, 0xa7, 0x8e, - 0xcc, 0xa2, 0xfb, 0x7f, 0x63, 0xb7, 0xa4, 0x75, 0xf2, 0x1e, 0x00, 0xbf, 0x44, 0x83, 0x72, 0x03, - 0xbf, 0xf7, 0xc5, 0x78, 0xd8, 0x3a, 0x2f, 0xdb, 0x4e, 0xc1, 0xfc, 0x26, 0x65, 0x82, 0xbf, 0x07, - 0xd3, 0x2d, 0xc1, 0x7b, 0xd4, 0x21, 0xb2, 0xf0, 0x13, 0x5e, 0x50, 0x7a, 0xaf, 0xd8, 0x5f, 0x42, - 0xe1, 0xcb, 0xca, 0xbc, 0x2b, 0xdc, 0xa6, 0x8e, 0x20, 0xdf, 0x80, 0xa8, 0xf8, 0x9c, 0x84, 0xe6, - 0x5f, 0xec, 0xfe, 0xd0, 0xc4, 0xd5, 0x5c, 0x3a, 0xeb, 0x15, 0x4a, 0x89, 0xa3, 0xb2, 0x15, 0x65, - 0xd1, 0xf3, 0x44, 0x78, 0x51, 0xa2, 0x4c, 0xdf, 0x07, 0x10, 0x71, 0x7f, 0xac, 0x46, 0x55, 0xad, - 0x50, 0x07, 0x7f, 0xc1, 0xee, 0xe9, 0x4c, 0x0c, 0xe5, 0x2f, 0x2a, 0x73, 0x9e, 0xfc, 0x1a, 0xe7, - 0x63, 0xa2, 0xef, 0x0e, 0x7f, 0x20, 0x17, 0x51, 0xdc, 0xac, 0x12, 0x66, 0xe2, 0xb0, 0x20, 0x32, - 0x41, 0xa5, 0xe7, 0x3b, 0xa4, 0xaf, 0xa0, 0xd0, 0x55, 0x65, 0x85, 0x09, 0x2d, 0x32, 0x2a, 0xaa, - 0xa5, 0xf8, 0x5b, 0xbf, 0x7b, 0x3f, 0x30, 0x25, 0xb9, 0xe1, 0x0f, 0xf2, 0x45, 0x14, 0xbc, 0x14, - 0x8b, 0xb6, 0xac, 0x4d, 0xfd, 0x88, 0xb5, 0x1e, 0x1f, 0xb9, 0x46, 0x3f, 0xcf, 0x19, 0x77, 0x8d, - 0x8e, 0x05, 0x8c, 0xae, 0x23, 0x8d, 0x60, 0xf4, 0xfb, 0xcf, 0x59, 0x07, 0x64, 0xd4, 0x42, 0xd6, - 0x3b, 0x3c, 0x20, 0x77, 0x86, 0xaa, 0x0f, 0xae, 0x1c, 0xd2, 0x29, 0x47, 0x7b, 0x41, 0x75, 0xc3, - 0x4d, 0x34, 0x42, 0xc4, 0x78, 0xf0, 0x40, 0xbc, 0x21, 0x91, 0x5b, 0x30, 0x71, 0x0f, 0xff, 0xdd, - 0x92, 0xf4, 0xf0, 0x34, 0xc6, 0xcf, 0x29, 0x27, 0xca, 0x1c, 0xd2, 0xd2, 0x71, 0xeb, 0xee, 0x7f, - 0xff, 0xcf, 0x9f, 0xaf, 0x4a, 0x9f, 0x7d, 0xbe, 0x2a, 0xfd, 0xe3, 0xf3, 0x55, 0xe9, 0xe3, 0xa7, - 0xab, 0x23, 0x9f, 0x3d, 0x5d, 0x1d, 0xf9, 0xeb, 0xd3, 0xd5, 0x91, 0xef, 0xbc, 0x56, 0xd6, 0x9d, - 0xc3, 0x7a, 0x31, 0x59, 0x32, 0xab, 0x29, 0xd5, 0xaa, 0xaa, 0x9a, 0x5a, 0xb3, 0xcc, 0x23, 0x5a, - 0x72, 0xdc, 0xaf, 0x94, 0xfb, 0xaf, 0x9e, 0x9f, 0x8e, 0x2e, 0x6e, 0x22, 0xb0, 0xcb, 0x97, 0x93, - 0xdb, 0x66, 0x72, 0xb3, 0xa6, 0x17, 0x27, 0xd0, 0x86, 0x37, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, - 0x4c, 0xf1, 0xa8, 0xd2, 0xd8, 0x2a, 0x00, 0x00, + // 3025 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0x4b, 0x6c, 0x1b, 0xd7, + 0xb9, 0xd6, 0x88, 0x7a, 0xf1, 0xa7, 0x1e, 0xd4, 0x11, 0x25, 0x8f, 0x68, 0x47, 0x94, 0x27, 0x89, + 0x43, 0x2b, 0xbe, 0x64, 0xac, 0xdc, 0xe0, 0xda, 0xbe, 0xb9, 0xd7, 0x15, 0x29, 0xda, 0x96, 0x6c, + 0xd3, 0x0a, 0x65, 0xe5, 0x51, 0x14, 0x65, 0x87, 0x9c, 0x23, 0x6a, 0x24, 0x72, 0x86, 0x99, 0x19, + 0xca, 0x55, 0x8b, 0x6c, 0x8a, 0x02, 0x5d, 0x74, 0x13, 0xb4, 0xcb, 0x02, 0x4d, 0x81, 0x76, 0x95, + 0xae, 0xbb, 0x29, 0x5a, 0xa0, 0xcb, 0x2e, 0x03, 0x74, 0xd3, 0x6e, 0x88, 0x22, 0x69, 0x1b, 0x80, + 0xbb, 0xee, 0xbb, 0x28, 0xce, 0x63, 0x66, 0xce, 0xf0, 0x4d, 0xd9, 0x4a, 0x36, 0xdd, 0x69, 0xbe, + 0xf3, 0x3f, 0xcf, 0xf9, 0xcf, 0xff, 0x38, 0x14, 0xc4, 0xea, 0x27, 0x95, 0xb4, 0x5a, 0xd7, 0xd3, + 0x76, 0xa3, 0x54, 0xd3, 0x9d, 0x54, 0xdd, 0x32, 0x1d, 0x13, 0x85, 0xd4, 0xba, 0x1e, 0xbf, 0x5c, + 0x31, 0xcd, 0x4a, 0x15, 0xa7, 0x29, 0x54, 0x6a, 0x1c, 0xa6, 0x71, 0xad, 0xee, 0x9c, 0x31, 0x8a, + 0x78, 0xa2, 0x7d, 0xd1, 0xd1, 0x6b, 0xd8, 0x76, 0xd4, 0x5a, 0x9d, 0x13, 0x28, 0x27, 0xb7, 0xec, + 0x94, 0x6e, 0x52, 0xd9, 0x65, 0xd3, 0xc2, 0xe9, 0xd3, 0x9b, 0xe9, 0x0a, 0x36, 0xb0, 0xa5, 0x3a, + 0x58, 0xe3, 0x34, 0x49, 0x81, 0xc6, 0xc0, 0xce, 0x33, 0xd3, 0x3a, 0xd1, 0x8d, 0x4a, 0x37, 0xca, + 0x2b, 0x5c, 0x1d, 0xa1, 0x54, 0x0d, 0xc3, 0x74, 0x54, 0x47, 0x37, 0x0d, 0x9b, 0xaf, 0x7a, 0x4e, + 0x1c, 0x61, 0xb5, 0xea, 0x1c, 0x31, 0x54, 0xf9, 0x24, 0x0c, 0xb1, 0x5d, 0xb3, 0xb4, 0x4f, 0x1d, + 0x2b, 0xe0, 0x0f, 0x1b, 0xd8, 0x76, 0x76, 0x1c, 0x5c, 0x43, 0x9b, 0x30, 0x53, 0xb7, 0x74, 0xd3, + 0xd2, 0x9d, 0x33, 0x59, 0x5a, 0x97, 0x92, 0x52, 0x66, 0xa5, 0xd5, 0x4c, 0x20, 0x17, 0xbb, 0x61, + 0xd6, 0x74, 0x87, 0xfa, 0x5a, 0xf0, 0xe8, 0xd0, 0x5b, 0x10, 0x36, 0xd4, 0x1a, 0xb6, 0xeb, 0x6a, + 0x19, 0xcb, 0xa1, 0x75, 0x29, 0x19, 0xce, 0x5c, 0x6a, 0x35, 0x13, 0x4b, 0x1e, 0x28, 0x70, 0xf9, + 0x94, 0xe8, 0x4d, 0x08, 0x97, 0xab, 0x3a, 0x36, 0x9c, 0xa2, 0xae, 0xc9, 0x33, 0x94, 0x8d, 0xea, + 0x62, 0xe0, 0x8e, 0x26, 0xea, 0x72, 0x31, 0xb4, 0x0f, 0x53, 0x55, 0xb5, 0x84, 0xab, 0xb6, 0x3c, + 0xb1, 0x1e, 0x4a, 0x46, 0x36, 0x5f, 0x4d, 0xa9, 0x75, 0x3d, 0xd5, 0xcd, 0x95, 0xd4, 0x23, 0x4a, + 0x97, 0x33, 0x1c, 0xeb, 0x2c, 0x13, 0x6b, 0x35, 0x13, 0x51, 0xc6, 0x28, 0x88, 0xe5, 0xa2, 0x50, + 0x05, 0x22, 0xc2, 0xc6, 0xc9, 0x93, 0x54, 0xf2, 0x46, 0x6f, 0xc9, 0x5b, 0x3e, 0x31, 0x13, 0xbf, + 0xda, 0x6a, 0x26, 0x96, 0x05, 0x11, 0x82, 0x0e, 0x51, 0x32, 0xfa, 0x91, 0x04, 0x31, 0x0b, 0x7f, + 0xd8, 0xd0, 0x2d, 0xac, 0x15, 0x0d, 0x53, 0xc3, 0x45, 0xee, 0xcc, 0x14, 0x55, 0x79, 0xb3, 0xb7, + 0xca, 0x02, 0xe7, 0xca, 0x9b, 0x1a, 0x16, 0x1d, 0x53, 0x5a, 0xcd, 0xc4, 0x15, 0xab, 0x63, 0xd1, + 0x37, 0x40, 0x96, 0x0a, 0xa8, 0x73, 0x1d, 0x3d, 0x81, 0x99, 0xba, 0xa9, 0x15, 0xed, 0x3a, 0x2e, + 0xcb, 0xe3, 0xeb, 0x52, 0x32, 0xb2, 0x79, 0x39, 0xc5, 0x22, 0x8e, 0xda, 0x40, 0xa2, 0x32, 0x75, + 0x7a, 0x33, 0xb5, 0x67, 0x6a, 0xfb, 0x75, 0x5c, 0xa6, 0xe7, 0xb9, 0x58, 0x67, 0x1f, 0x01, 0xd9, + 0xd3, 0x1c, 0x44, 0x7b, 0x10, 0x76, 0x05, 0xda, 0xf2, 0x34, 0x75, 0xa7, 0xaf, 0x44, 0x16, 0x56, + 0xec, 0xc3, 0x0e, 0x84, 0x15, 0xc7, 0x50, 0x16, 0xa6, 0x75, 0xa3, 0x62, 0x61, 0xdb, 0x96, 0xc3, + 0x54, 0x1e, 0xa2, 0x82, 0x76, 0x18, 0x96, 0x35, 0x8d, 0x43, 0xbd, 0x92, 0x59, 0x26, 0x86, 0x71, + 0x32, 0x41, 0x8a, 0xcb, 0x89, 0xee, 0xc1, 0x8c, 0x8d, 0xad, 0x53, 0xbd, 0x8c, 0x6d, 0x19, 0x04, + 0x29, 0xfb, 0x0c, 0xe4, 0x52, 0xa8, 0x31, 0x2e, 0x9d, 0x68, 0x8c, 0x8b, 0x91, 0x18, 0xb7, 0xcb, + 0x47, 0x58, 0x6b, 0x54, 0xb1, 0x25, 0x47, 0xfc, 0x18, 0xf7, 0x40, 0x31, 0xc6, 0x3d, 0x30, 0xae, + 0x42, 0x44, 0x38, 0x2d, 0xf4, 0x32, 0x84, 0x4e, 0x30, 0xbb, 0x58, 0xe1, 0xcc, 0x62, 0xab, 0x99, + 0x98, 0x3b, 0xc1, 0xe2, 0x9d, 0x22, 0xab, 0xe8, 0x3a, 0x4c, 0x9e, 0xaa, 0xd5, 0x06, 0xa6, 0xe7, + 0x12, 0xce, 0x2c, 0xb5, 0x9a, 0x89, 0x05, 0x0a, 0x08, 0x84, 0x8c, 0xe2, 0xce, 0xf8, 0x2d, 0x29, + 0x7e, 0x08, 0xd1, 0xf6, 0x78, 0xbc, 0x10, 0x3d, 0x35, 0xb8, 0xd4, 0x23, 0x08, 0x2f, 0x42, 0x9d, + 0xf2, 0xcf, 0x10, 0xcc, 0x05, 0x8e, 0x1a, 0xdd, 0x81, 0x09, 0xe7, 0xac, 0x8e, 0xa9, 0x9a, 0xf9, + 0xcd, 0xa8, 0x18, 0x0c, 0x4f, 0xcf, 0xea, 0x98, 0xde, 0xf1, 0x79, 0x42, 0x11, 0x08, 0x50, 0xca, + 0x43, 0x94, 0xd7, 0x4d, 0xcb, 0xb1, 0xe5, 0xf1, 0xf5, 0x50, 0x72, 0x8e, 0x29, 0xa7, 0x80, 0xa8, + 0x9c, 0x02, 0xe8, 0x3b, 0xc1, 0x64, 0x10, 0xa2, 0x41, 0xf3, 0x72, 0x67, 0xe8, 0x9d, 0x3f, 0x0b, + 0xdc, 0x86, 0x88, 0x53, 0xb5, 0x8b, 0xd8, 0x50, 0x4b, 0x55, 0xac, 0xc9, 0x13, 0xeb, 0x52, 0x72, + 0x26, 0x23, 0xb7, 0x9a, 0x89, 0x98, 0x43, 0x76, 0x94, 0xa2, 0x02, 0x2f, 0xf8, 0x28, 0xcd, 0x99, + 0xd8, 0x72, 0x8a, 0x24, 0x8b, 0xca, 0x93, 0x42, 0xce, 0xc4, 0x96, 0x93, 0x57, 0x6b, 0x38, 0x90, + 0x33, 0x39, 0x86, 0xee, 0xc2, 0x5c, 0xc3, 0xc6, 0xc5, 0x72, 0xb5, 0x61, 0x3b, 0xd8, 0xda, 0xd9, + 0x93, 0xa7, 0xa8, 0xc6, 0x78, 0xab, 0x99, 0x58, 0x69, 0xd8, 0x38, 0xeb, 0xe2, 0x02, 0xf3, 0xac, + 0x88, 0x7f, 0x55, 0x21, 0xa6, 0x38, 0x30, 0x17, 0xb8, 0x97, 0xe8, 0x56, 0x97, 0x23, 0xe7, 0x14, + 0xf4, 0xc8, 0x51, 0xe7, 0x91, 0x8f, 0x7c, 0xe0, 0xca, 0x5f, 0x24, 0x88, 0xb6, 0xe7, 0x5c, 0xc2, + 0xff, 0x61, 0x03, 0x37, 0x30, 0x77, 0x90, 0xf2, 0x53, 0x40, 0xe4, 0xa7, 0x00, 0xfa, 0x6f, 0x80, + 0x63, 0xb3, 0x54, 0xb4, 0x31, 0x2d, 0x64, 0xe3, 0xfe, 0xa1, 0x1c, 0x9b, 0xa5, 0x7d, 0xdc, 0x56, + 0xc8, 0x5c, 0x0c, 0x69, 0xb0, 0x48, 0xb8, 0x2c, 0xa6, 0xaf, 0x48, 0x08, 0xdc, 0x60, 0x5b, 0xed, + 0x59, 0x06, 0x32, 0x2f, 0xb5, 0x9a, 0x89, 0xd5, 0x63, 0xb3, 0x24, 0x60, 0xa2, 0x47, 0x0b, 0x6d, + 0x4b, 0xca, 0x2f, 0x25, 0x58, 0xdc, 0x35, 0x4b, 0x7b, 0x16, 0x26, 0x04, 0x5f, 0x99, 0x73, 0xff, + 0x05, 0xd3, 0x84, 0x4b, 0xd7, 0x98, 0x4b, 0x61, 0x56, 0x7f, 0x8f, 0xcd, 0xd2, 0x8e, 0x16, 0xa8, + 0xbf, 0x0c, 0x51, 0xfe, 0xc5, 0x4e, 0x20, 0xab, 0x1a, 0x65, 0x5c, 0x75, 0x8d, 0xdc, 0x80, 0x29, + 0x26, 0x43, 0xb4, 0x92, 0x32, 0x88, 0x56, 0x52, 0xe0, 0x9c, 0x56, 0x7a, 0xdb, 0x10, 0x1a, 0xb8, + 0x0d, 0x82, 0x43, 0x13, 0x83, 0x1d, 0x42, 0x37, 0x60, 0xca, 0xc2, 0xaa, 0x6d, 0x1a, 0xfc, 0x8e, + 0x52, 0x6a, 0x86, 0x88, 0xd4, 0x0c, 0x51, 0xfe, 0x2e, 0xc1, 0xd2, 0x2e, 0x35, 0x2a, 0xb8, 0x03, + 0x41, 0xaf, 0xa4, 0x51, 0xbd, 0x1a, 0x1f, 0xe8, 0xd5, 0x5d, 0x98, 0x3a, 0xd4, 0xab, 0x0e, 0xb6, + 0xe8, 0x0e, 0x44, 0x36, 0x17, 0xbd, 0xc0, 0xc3, 0xce, 0x3d, 0xba, 0xc0, 0x2c, 0x67, 0x44, 0xa2, + 0xe5, 0x0c, 0x11, 0xfc, 0x9c, 0x18, 0xc2, 0xcf, 0x87, 0x30, 0x2b, 0xca, 0x46, 0xff, 0x0b, 0x53, + 0xb6, 0xa3, 0x3a, 0xd8, 0x96, 0xa5, 0xf5, 0x50, 0x72, 0x7e, 0x73, 0xce, 0x53, 0x4f, 0x50, 0x26, + 0x8c, 0x11, 0x88, 0xc2, 0x18, 0xa2, 0x7c, 0xb9, 0x00, 0xa1, 0x5d, 0xb3, 0x84, 0xd6, 0x61, 0xdc, + 0xdb, 0x9c, 0x68, 0xab, 0x99, 0x98, 0xd5, 0xc5, 0x6d, 0x19, 0xd7, 0xb5, 0x60, 0x9f, 0x39, 0x37, + 0x64, 0x9f, 0x79, 0xe1, 0x11, 0x15, 0x68, 0x9a, 0xa7, 0x87, 0x6e, 0x9a, 0x33, 0x5e, 0xff, 0xcb, + 0x7a, 0xa2, 0x98, 0xbb, 0x67, 0x23, 0xb4, 0xbb, 0xef, 0x06, 0x2b, 0x1c, 0x04, 0x93, 0xce, 0xf9, + 0xeb, 0xda, 0x69, 0x8f, 0xe6, 0x36, 0x42, 0x15, 0xac, 0x7b, 0x0a, 0x5e, 0x74, 0x2f, 0x7b, 0x1d, + 0x26, 0xcd, 0x67, 0x06, 0xb6, 0xf8, 0x10, 0x41, 0x77, 0x9d, 0x02, 0xe2, 0xae, 0x53, 0x00, 0x61, + 0xb8, 0x4c, 0xb7, 0xbf, 0x48, 0x3f, 0xed, 0x23, 0xbd, 0x5e, 0x6c, 0xd8, 0xd8, 0x2a, 0x56, 0x2c, + 0xb3, 0x51, 0xb7, 0xe5, 0x05, 0x7a, 0xb7, 0xaf, 0xb5, 0x9a, 0x09, 0x85, 0x92, 0x3d, 0x71, 0xa9, + 0x0e, 0x6c, 0x6c, 0xdd, 0xa7, 0x34, 0x82, 0x4c, 0xb9, 0x17, 0x0d, 0xfa, 0xa1, 0x04, 0xd7, 0xca, + 0x66, 0xad, 0x4e, 0xba, 0x05, 0xac, 0x15, 0xfb, 0xa9, 0x5c, 0x5a, 0x97, 0x92, 0xb3, 0x99, 0x37, + 0x5a, 0xcd, 0xc4, 0x0d, 0x9f, 0xe3, 0x9d, 0xc1, 0xca, 0x95, 0xc1, 0xd4, 0x81, 0x61, 0x6e, 0x62, + 0xc8, 0x61, 0x4e, 0x1c, 0x0c, 0x26, 0x5f, 0xf8, 0x60, 0x30, 0xfb, 0x22, 0x06, 0x83, 0x5f, 0x48, + 0xb0, 0xce, 0x5b, 0x6c, 0xdd, 0xa8, 0x14, 0x2d, 0x6c, 0x9b, 0x0d, 0xab, 0x8c, 0x8b, 0x3c, 0x34, + 0x6a, 0xd8, 0x70, 0x6c, 0x79, 0x99, 0xda, 0x9e, 0xec, 0xa6, 0xa9, 0xc0, 0x19, 0x0a, 0x02, 0x7d, + 0xe6, 0x46, 0xab, 0x99, 0x48, 0xfa, 0x52, 0xbb, 0xd1, 0x08, 0xc6, 0xac, 0xf5, 0xa7, 0x44, 0x0f, + 0x61, 0xba, 0x6c, 0x61, 0x32, 0xa4, 0xd3, 0x66, 0x2b, 0xb2, 0x19, 0x4f, 0xb1, 0x29, 0x3d, 0xe5, + 0x3e, 0x0a, 0xa4, 0x9e, 0xba, 0x8f, 0x02, 0x6c, 0x86, 0xe1, 0xe4, 0xe2, 0x0c, 0xc3, 0x21, 0x71, + 0x10, 0x9a, 0x7f, 0x21, 0x83, 0x50, 0xf4, 0x39, 0x06, 0xa1, 0x6f, 0x41, 0xe4, 0xe4, 0x96, 0x5d, + 0x74, 0x0d, 0x5a, 0xa4, 0xa2, 0xae, 0x8a, 0xdb, 0xec, 0xbf, 0x56, 0x90, 0xcd, 0xe6, 0x56, 0xb2, + 0xfe, 0xf6, 0xe4, 0x96, 0xbd, 0xd3, 0x61, 0x22, 0xf8, 0x28, 0x49, 0x4d, 0x44, 0x3a, 0xd7, 0x26, + 0xa3, 0xde, 0xe1, 0xc2, 0xed, 0xf6, 0xe4, 0xf2, 0xef, 0x36, 0xb9, 0x1c, 0x0d, 0x8e, 0x6f, 0xb1, + 0x61, 0xc7, 0x37, 0xb4, 0x03, 0x8b, 0xec, 0xee, 0x3a, 0x4e, 0xb5, 0x68, 0xe3, 0xb2, 0x69, 0x68, + 0xb6, 0xbc, 0xb2, 0x2e, 0x25, 0x43, 0xac, 0x13, 0xa3, 0x8b, 0x4f, 0x9d, 0xea, 0x3e, 0x5b, 0x12, + 0x3b, 0xb1, 0xb6, 0xa5, 0xff, 0x4c, 0x82, 0xe7, 0x9e, 0x0a, 0xfe, 0x21, 0xc1, 0xca, 0x2e, 0xe9, + 0x6b, 0x79, 0x8e, 0xd2, 0xbf, 0x87, 0xdd, 0x0e, 0x49, 0x68, 0xcb, 0xa4, 0x21, 0xda, 0xb2, 0x0b, + 0x2f, 0xea, 0x6f, 0xc3, 0xac, 0x81, 0x9f, 0x15, 0xdb, 0x92, 0x2e, 0xad, 0x9f, 0x06, 0x7e, 0xb6, + 0xd7, 0x99, 0x77, 0x23, 0x02, 0xac, 0xfc, 0x7a, 0x1c, 0x2e, 0x75, 0x38, 0x6a, 0xd7, 0x4d, 0xc3, + 0xc6, 0xe8, 0x67, 0x12, 0xc8, 0x96, 0xbf, 0x40, 0x8f, 0x98, 0x64, 0xbe, 0x46, 0xd5, 0x61, 0xbe, + 0x47, 0x36, 0x6f, 0xbb, 0x05, 0xb6, 0x9b, 0x80, 0x54, 0xa1, 0x8d, 0xb9, 0xc0, 0x78, 0x59, 0xe5, + 0x7d, 0xb5, 0xd5, 0x4c, 0x5c, 0xb5, 0xba, 0x53, 0x08, 0xd6, 0x5e, 0xea, 0x41, 0x12, 0xb7, 0xe0, + 0x4a, 0x3f, 0xf9, 0x17, 0x12, 0x16, 0x06, 0x2c, 0x0b, 0x23, 0x12, 0xf3, 0x92, 0x3e, 0x61, 0x8e, + 0x32, 0x38, 0x5c, 0x87, 0x49, 0x6c, 0x59, 0xa6, 0x25, 0xea, 0xa4, 0x80, 0x48, 0x4a, 0x01, 0xe5, + 0x23, 0x3a, 0x49, 0x05, 0xf5, 0xa1, 0x23, 0x40, 0x6c, 0x8a, 0x63, 0xdf, 0x7c, 0x8c, 0x63, 0xe7, + 0x11, 0x6f, 0x1f, 0xe3, 0x7c, 0x1b, 0x33, 0x6b, 0xad, 0x66, 0x22, 0x4e, 0x87, 0x35, 0x1f, 0x14, + 0x77, 0x3a, 0xda, 0xbe, 0xa6, 0xfc, 0x3e, 0x0c, 0x93, 0xb4, 0xd0, 0xa3, 0x6b, 0x30, 0x41, 0xc7, + 0x7f, 0xe6, 0x1d, 0x1d, 0x81, 0x8d, 0xe0, 0xe8, 0x4f, 0xd7, 0x51, 0x0e, 0x16, 0xdc, 0x40, 0x2c, + 0x1e, 0xaa, 0x65, 0x87, 0x7b, 0x29, 0x65, 0xae, 0xb4, 0x9a, 0x09, 0xd9, 0x5d, 0xba, 0x47, 0x57, + 0x04, 0xe6, 0xf9, 0xe0, 0x0a, 0xba, 0x0d, 0x11, 0xda, 0xaf, 0xb0, 0xf6, 0x85, 0xcf, 0x73, 0x34, + 0xeb, 0x12, 0x98, 0xb5, 0x1d, 0x62, 0xd6, 0xf5, 0x51, 0x72, 0x1d, 0x68, 0x97, 0xe3, 0xf2, 0xb2, + 0xd1, 0x89, 0x5e, 0x07, 0x8a, 0x77, 0x30, 0x47, 0x04, 0x18, 0x55, 0x60, 0xc1, 0x2b, 0xed, 0x55, + 0xbd, 0xa6, 0x3b, 0xee, 0xcb, 0xec, 0x1a, 0xdd, 0x58, 0xba, 0x19, 0x5e, 0x2d, 0x7f, 0x44, 0x09, + 0x58, 0x34, 0x93, 0xcd, 0x95, 0xad, 0xc0, 0x42, 0xa0, 0x35, 0x99, 0x0f, 0xae, 0xa1, 0xdf, 0x48, + 0x70, 0xad, 0x4d, 0x53, 0xb1, 0x74, 0xe6, 0xdd, 0xe2, 0x62, 0xb9, 0xaa, 0xda, 0x36, 0x7b, 0x72, + 0x99, 0x16, 0xde, 0x69, 0xbb, 0x19, 0x90, 0x39, 0x73, 0x6f, 0x73, 0x96, 0x30, 0xe5, 0xd5, 0x1a, + 0x66, 0x36, 0xa5, 0x5b, 0xcd, 0xc4, 0xeb, 0xd6, 0x20, 0x5a, 0x61, 0x2b, 0xae, 0x0e, 0x24, 0x46, + 0xfb, 0x10, 0xa9, 0x63, 0xab, 0xa6, 0xdb, 0x36, 0xed, 0xe3, 0xd9, 0x1b, 0xf2, 0x8a, 0x60, 0xdb, + 0x9e, 0xbf, 0xca, 0x76, 0x5d, 0x20, 0x17, 0x77, 0x5d, 0x80, 0x49, 0xcf, 0x58, 0x36, 0x2d, 0xcd, + 0x34, 0x30, 0x7b, 0x94, 0x9f, 0xe1, 0xc3, 0x12, 0xc7, 0x02, 0xc3, 0x12, 0xc7, 0xc8, 0x18, 0x28, + 0x0c, 0x25, 0xe1, 0xfe, 0xe3, 0x47, 0xfc, 0x4b, 0x09, 0x22, 0x82, 0x65, 0xa8, 0x00, 0x33, 0x76, + 0xa3, 0x74, 0x8c, 0xcb, 0x5e, 0x26, 0x5b, 0xeb, 0xee, 0x43, 0x6a, 0x9f, 0x91, 0xf1, 0x2e, 0x85, + 0xf3, 0x04, 0xba, 0x14, 0x8e, 0xd1, 0x5c, 0x82, 0xad, 0x12, 0x7b, 0xfe, 0x71, 0x73, 0x09, 0x01, + 0x02, 0xb9, 0x84, 0x00, 0xf1, 0x0f, 0x60, 0x9a, 0xcb, 0x25, 0x37, 0xeb, 0x44, 0x37, 0x34, 0xf1, + 0x66, 0x91, 0x6f, 0xf1, 0x66, 0x91, 0x6f, 0xef, 0x06, 0x8e, 0xf7, 0xbf, 0x81, 0x71, 0x1d, 0x96, + 0xba, 0xc4, 0xe7, 0x39, 0xb2, 0xa1, 0x34, 0xb0, 0x26, 0xff, 0x5c, 0x82, 0x6b, 0xc3, 0x85, 0xe2, + 0x70, 0xea, 0x1f, 0x8a, 0xea, 0xdd, 0xe1, 0x2d, 0x20, 0xb0, 0x4d, 0xdb, 0xa0, 0x74, 0xfd, 0x93, + 0x49, 0xb8, 0xdc, 0x87, 0x9f, 0x34, 0xf5, 0xab, 0x35, 0xf5, 0xbb, 0x7a, 0xad, 0x51, 0xf3, 0x3b, + 0xfa, 0x43, 0x4b, 0x2d, 0x93, 0x62, 0xc2, 0xe3, 0xe2, 0xff, 0x06, 0x59, 0x91, 0x7a, 0xcc, 0x24, + 0xb8, 0xe8, 0x3d, 0xce, 0x2f, 0x54, 0xb9, 0x5a, 0x77, 0x0a, 0xb1, 0xca, 0xf5, 0x20, 0x41, 0xbf, + 0x95, 0xe0, 0x6a, 0x4f, 0x13, 0x69, 0xc6, 0x30, 0xcd, 0x2a, 0x8d, 0xb8, 0xc8, 0x66, 0xf6, 0xbc, + 0xa6, 0x66, 0xce, 0xf6, 0x4c, 0xb3, 0xca, 0x0c, 0x7e, 0xbd, 0xd5, 0x4c, 0xbc, 0x56, 0xeb, 0x47, + 0x27, 0x98, 0xfd, 0x52, 0x5f, 0x42, 0x52, 0xa2, 0xfb, 0x6d, 0xce, 0x45, 0x05, 0xa5, 0x32, 0xd8, + 0xcd, 0xe1, 0x54, 0x3f, 0x09, 0x06, 0xe4, 0x2b, 0x9d, 0xfb, 0x4b, 0x04, 0x8e, 0x18, 0x94, 0xbf, + 0x1b, 0x87, 0xc4, 0x00, 0x19, 0xe8, 0x57, 0x43, 0x04, 0xe6, 0xd6, 0x30, 0xd6, 0x5c, 0x68, 0x70, + 0x7e, 0x1d, 0xe7, 0xab, 0xe4, 0x20, 0x4c, 0x93, 0xf4, 0x23, 0xdd, 0x76, 0xd0, 0x2d, 0x98, 0xa2, + 0x4d, 0xb0, 0x9b, 0xc4, 0xc1, 0x4f, 0xe2, 0xac, 0x20, 0xb0, 0x55, 0xb1, 0x20, 0x30, 0x44, 0x39, + 0x00, 0xc4, 0x1e, 0x3e, 0xab, 0x42, 0xe7, 0x88, 0xee, 0xc2, 0x5c, 0x99, 0xa1, 0x58, 0x13, 0x3a, + 0x7c, 0xfa, 0xab, 0x85, 0xb7, 0x10, 0xec, 0xf3, 0x67, 0x45, 0x5c, 0xb9, 0x0d, 0x0b, 0x54, 0xfb, + 0x7d, 0xec, 0x3d, 0x7c, 0x0f, 0xd9, 0x3a, 0x29, 0x6f, 0x03, 0xa2, 0xac, 0x59, 0x5a, 0xe1, 0x46, + 0xe5, 0xfe, 0x7f, 0x88, 0x51, 0xee, 0x03, 0xa3, 0x7c, 0x2e, 0xfe, 0xbb, 0x20, 0xef, 0x3b, 0x16, + 0x56, 0x6b, 0xba, 0x51, 0x69, 0xf7, 0xe0, 0x65, 0x08, 0x19, 0x8d, 0x1a, 0x15, 0x31, 0xc7, 0x8e, + 0xd1, 0x68, 0xd4, 0xc4, 0x63, 0x34, 0x1a, 0x35, 0xcf, 0xfc, 0x6d, 0x5c, 0xc5, 0x0e, 0x1e, 0x55, + 0xfd, 0xa7, 0x12, 0x00, 0x7b, 0xa7, 0xdd, 0x31, 0x0e, 0xcd, 0xa1, 0xdb, 0xcd, 0xdb, 0x10, 0xa1, + 0xe7, 0xa9, 0x15, 0x8f, 0x4d, 0x5a, 0x78, 0xa5, 0xe4, 0x24, 0xeb, 0x13, 0x19, 0xbc, 0x6b, 0x06, + 0xaa, 0x2f, 0xf8, 0x28, 0x61, 0xad, 0x62, 0xd5, 0x76, 0x59, 0x43, 0x3e, 0x2b, 0x83, 0xdb, 0x59, + 0x7d, 0x54, 0x79, 0x06, 0x4b, 0x6c, 0xaf, 0xeb, 0x9a, 0xea, 0xf8, 0xe3, 0xd2, 0x5b, 0xe2, 0x2f, + 0x1c, 0xc1, 0x58, 0xec, 0x37, 0xbf, 0x8d, 0x30, 0x0e, 0x34, 0x40, 0xce, 0xa8, 0x4e, 0xf9, 0xa8, + 0x9b, 0xf6, 0x0f, 0x60, 0xee, 0x50, 0xd5, 0xab, 0xee, 0xcb, 0x9f, 0x7b, 0x23, 0x64, 0xdf, 0x8a, + 0x20, 0x03, 0x0b, 0x6a, 0xc6, 0xf2, 0x4e, 0xfb, 0x2d, 0x99, 0x15, 0x71, 0xcf, 0xdf, 0x2c, 0x7d, + 0x1b, 0xfa, 0xba, 0xfc, 0x6d, 0xd3, 0x3e, 0xd8, 0xdf, 0x20, 0xc3, 0x08, 0xfe, 0x46, 0x20, 0x9c, + 0x33, 0xb4, 0xc7, 0xaa, 0x75, 0x82, 0x2d, 0xe5, 0x63, 0x09, 0x96, 0x83, 0x37, 0xe3, 0x31, 0xb6, + 0x6d, 0xb5, 0x82, 0xd1, 0xff, 0x8c, 0xe6, 0xff, 0x83, 0x31, 0xff, 0x19, 0x3e, 0x84, 0x0d, 0x8d, + 0x17, 0x95, 0x79, 0xca, 0xe6, 0xe9, 0x63, 0xf7, 0x0b, 0x8b, 0x0d, 0xe0, 0x83, 0xb1, 0x02, 0xa1, + 0xcf, 0x4c, 0xc3, 0x24, 0x3e, 0xc5, 0x86, 0xb3, 0x11, 0x87, 0x88, 0xf0, 0x1b, 0x34, 0x8a, 0xc0, + 0x34, 0xff, 0x8c, 0x8e, 0x6d, 0x5c, 0x87, 0x88, 0xf0, 0x63, 0x25, 0x9a, 0x85, 0x99, 0xbc, 0xa9, + 0xe1, 0x3d, 0xd3, 0x72, 0xa2, 0x63, 0xe4, 0xeb, 0x01, 0x56, 0xb5, 0x2a, 0x21, 0x95, 0x36, 0x3e, + 0x91, 0x60, 0xc6, 0xfd, 0xe1, 0x03, 0x01, 0x4c, 0xbd, 0x73, 0x90, 0x3b, 0xc8, 0x6d, 0x47, 0xc7, + 0x88, 0xc0, 0xbd, 0x5c, 0x7e, 0x7b, 0x27, 0x7f, 0x3f, 0x2a, 0x91, 0x8f, 0xc2, 0x41, 0x3e, 0x4f, + 0x3e, 0xc6, 0xd1, 0x1c, 0x84, 0xf7, 0x0f, 0xb2, 0xd9, 0x5c, 0x6e, 0x3b, 0xb7, 0x1d, 0x0d, 0x11, + 0xa6, 0x7b, 0x5b, 0x3b, 0x8f, 0x72, 0xdb, 0xd1, 0x09, 0x42, 0x77, 0x90, 0x7f, 0x98, 0x7f, 0xf2, + 0x5e, 0x3e, 0x3a, 0xc9, 0xe8, 0x32, 0x8f, 0x77, 0x9e, 0x3e, 0xcd, 0x6d, 0x47, 0xa7, 0x08, 0xdd, + 0xa3, 0xdc, 0xd6, 0x7e, 0x6e, 0x3b, 0x3a, 0x4d, 0x96, 0xf6, 0x0a, 0xb9, 0xdc, 0xe3, 0x3d, 0xb2, + 0x34, 0x43, 0x3e, 0xb3, 0x5b, 0xf9, 0x6c, 0xee, 0x11, 0x91, 0x12, 0x26, 0x16, 0x16, 0x72, 0xbb, + 0xb9, 0x2c, 0x59, 0x84, 0xcd, 0x3f, 0x4c, 0xc0, 0x2c, 0xdd, 0x50, 0xf7, 0x49, 0xed, 0x4d, 0x88, + 0xb0, 0x53, 0x65, 0x53, 0xa9, 0xb0, 0xe5, 0xf1, 0x95, 0x8e, 0xc7, 0xce, 0x1c, 0xd9, 0x3c, 0x65, + 0x0c, 0xdd, 0x85, 0x59, 0x81, 0xc9, 0x46, 0xf3, 0x3e, 0x17, 0x29, 0x22, 0xf1, 0x97, 0xe8, 0x77, + 0xaf, 0x40, 0x53, 0xc6, 0x88, 0x56, 0x76, 0x77, 0x46, 0xd4, 0x2a, 0x30, 0x0d, 0xd6, 0x1a, 0xbc, + 0x9d, 0xca, 0x18, 0xfa, 0x06, 0x44, 0x58, 0x2e, 0x65, 0x5a, 0x2f, 0xf9, 0xfc, 0x81, 0x14, 0xdb, + 0xc7, 0x84, 0x14, 0xcc, 0xdc, 0xc7, 0x0e, 0x63, 0x8f, 0xf9, 0xec, 0x7e, 0x66, 0x8f, 0x0b, 0xae, + 0x28, 0x63, 0x68, 0x17, 0xc2, 0x2e, 0xbd, 0x8d, 0x98, 0x7d, 0xbd, 0x6a, 0x42, 0x3c, 0xde, 0x65, + 0x99, 0x5f, 0x0c, 0x65, 0xec, 0x0d, 0x89, 0x58, 0xcf, 0x0a, 0x59, 0x87, 0xf5, 0x81, 0xfa, 0xd6, + 0xc7, 0xfa, 0x6d, 0x98, 0x73, 0x8b, 0x19, 0x93, 0xb1, 0x2a, 0xa4, 0xb2, 0x60, 0x95, 0xeb, 0x2d, + 0x65, 0xf3, 0xc7, 0x61, 0x98, 0x62, 0x6f, 0x21, 0xe8, 0x5d, 0x00, 0xf6, 0x17, 0xcd, 0xff, 0xcb, + 0x5d, 0x7f, 0xf0, 0x8e, 0xaf, 0x74, 0x7f, 0x40, 0x51, 0x56, 0x7f, 0xf0, 0xa7, 0xbf, 0xfd, 0x74, + 0x7c, 0x49, 0x99, 0x4f, 0x9f, 0xde, 0x4c, 0x1f, 0x9b, 0x25, 0xfe, 0xff, 0x79, 0x77, 0xa4, 0x0d, + 0xf4, 0x1e, 0x00, 0x6b, 0x25, 0x82, 0x72, 0x03, 0xbf, 0xab, 0xc6, 0xd9, 0x06, 0x74, 0xb6, 0x1c, + 0x9d, 0x82, 0x59, 0x3f, 0x41, 0x04, 0x7f, 0x1b, 0x66, 0x3d, 0xc1, 0xfb, 0xd8, 0x41, 0xb2, 0xf0, + 0x53, 0x69, 0x50, 0x7a, 0x2f, 0xff, 0xaf, 0x50, 0xe1, 0x2b, 0xca, 0x22, 0x17, 0x6e, 0x63, 0x47, + 0x90, 0x6f, 0x40, 0x54, 0x7c, 0xb6, 0xa3, 0xe6, 0x5f, 0xee, 0xfe, 0xa0, 0xc7, 0xd4, 0x5c, 0xe9, + 0xf7, 0xda, 0xa7, 0x24, 0xa8, 0xb2, 0x55, 0x25, 0xe6, 0x7a, 0x22, 0xbc, 0xdc, 0x61, 0xa2, 0xef, + 0x03, 0x88, 0xf0, 0x7f, 0x0a, 0xa0, 0xaa, 0xbc, 0xad, 0x0e, 0xfe, 0xa7, 0x40, 0x4f, 0x67, 0xe2, + 0x54, 0x7e, 0x4c, 0x59, 0x70, 0xe5, 0xd7, 0x19, 0x1f, 0x11, 0x7d, 0x7f, 0xf4, 0xc4, 0x10, 0xa3, + 0xe2, 0xe6, 0x95, 0x30, 0x11, 0x47, 0x13, 0x33, 0x11, 0x54, 0x7e, 0xbe, 0x64, 0xf1, 0x0a, 0x15, + 0xba, 0xa6, 0xac, 0x12, 0xa1, 0x25, 0x42, 0x85, 0xb5, 0x34, 0xfb, 0x4d, 0x85, 0xd7, 0x29, 0xa2, + 0x24, 0x3f, 0x7a, 0x42, 0xb9, 0x4c, 0x05, 0x2f, 0xc7, 0xa3, 0x9e, 0xb5, 0xe9, 0xef, 0x93, 0x16, + 0xe8, 0x23, 0x6e, 0xf4, 0xf3, 0xe4, 0x1a, 0x6e, 0x74, 0x3c, 0x60, 0x74, 0x83, 0xd2, 0x08, 0x46, + 0xbf, 0xff, 0x9c, 0xf9, 0x48, 0xa6, 0x5a, 0xd0, 0x46, 0x87, 0x07, 0xe8, 0xde, 0x48, 0x79, 0x8a, + 0xcb, 0x41, 0x9d, 0x72, 0xb4, 0x17, 0x94, 0xbf, 0x78, 0xa0, 0x21, 0x24, 0xee, 0x07, 0xdb, 0x88, + 0x37, 0x24, 0x74, 0x07, 0xa6, 0x1e, 0xd0, 0x7f, 0x6b, 0x45, 0x3d, 0x3c, 0x8d, 0xb3, 0x7b, 0xca, + 0x88, 0xb2, 0x47, 0xb8, 0x7c, 0xe2, 0xf5, 0x20, 0xef, 0xff, 0xf1, 0xf3, 0x35, 0xe9, 0xb3, 0xcf, + 0xd7, 0xa4, 0xbf, 0x7e, 0xbe, 0x26, 0x7d, 0xfc, 0xc5, 0xda, 0xd8, 0x67, 0x5f, 0xac, 0x8d, 0xfd, + 0xf9, 0x8b, 0xb5, 0xb1, 0x6f, 0xbe, 0x56, 0xd1, 0x9d, 0xa3, 0x46, 0x29, 0x55, 0x36, 0x6b, 0x69, + 0xd5, 0xaa, 0xa9, 0x9a, 0x5a, 0xb7, 0xcc, 0x63, 0x5c, 0x76, 0xf8, 0x57, 0x9a, 0xff, 0x4b, 0xed, + 0xa7, 0xe3, 0xb1, 0x2d, 0x0a, 0xec, 0xb1, 0xe5, 0xd4, 0x8e, 0x99, 0xda, 0xaa, 0xeb, 0xa5, 0x29, + 0x6a, 0xc3, 0x9b, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x8f, 0x40, 0x05, 0xa7, 0x40, 0x2c, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2314,6 +2429,8 @@ type QueueServiceClient interface { DeleteQueue(ctx context.Context, in *QueueDeleteRequest, opts ...grpc.CallOption) (*types.Empty, error) GetQueue(ctx context.Context, in *QueueGetRequest, opts ...grpc.CallOption) (*Queue, error) GetQueues(ctx context.Context, in *StreamingQueueGetRequest, opts ...grpc.CallOption) (QueueService_GetQueuesClient, error) + CordonQueue(ctx context.Context, in *QueueCordonRequest, opts ...grpc.CallOption) (*types.Empty, error) + UncordonQueue(ctx context.Context, in *QueueUncordonRequest, opts ...grpc.CallOption) (*types.Empty, error) } type queueServiceClient struct { @@ -2410,6 +2527,24 @@ func (x *queueServiceGetQueuesClient) Recv() (*StreamingQueueMessage, error) { return m, nil } +func (c *queueServiceClient) CordonQueue(ctx context.Context, in *QueueCordonRequest, opts ...grpc.CallOption) (*types.Empty, error) { + out := new(types.Empty) + err := c.cc.Invoke(ctx, "/api.QueueService/CordonQueue", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *queueServiceClient) UncordonQueue(ctx context.Context, in *QueueUncordonRequest, opts ...grpc.CallOption) (*types.Empty, error) { + out := new(types.Empty) + err := c.cc.Invoke(ctx, "/api.QueueService/UncordonQueue", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // QueueServiceServer is the server API for QueueService service. type QueueServiceServer interface { CreateQueue(context.Context, *Queue) (*types.Empty, error) @@ -2419,6 +2554,8 @@ type QueueServiceServer interface { DeleteQueue(context.Context, *QueueDeleteRequest) (*types.Empty, error) GetQueue(context.Context, *QueueGetRequest) (*Queue, error) GetQueues(*StreamingQueueGetRequest, QueueService_GetQueuesServer) error + CordonQueue(context.Context, *QueueCordonRequest) (*types.Empty, error) + UncordonQueue(context.Context, *QueueUncordonRequest) (*types.Empty, error) } // UnimplementedQueueServiceServer can be embedded to have forward compatible implementations. @@ -2446,6 +2583,12 @@ func (*UnimplementedQueueServiceServer) GetQueue(ctx context.Context, req *Queue func (*UnimplementedQueueServiceServer) GetQueues(req *StreamingQueueGetRequest, srv QueueService_GetQueuesServer) error { return status.Errorf(codes.Unimplemented, "method GetQueues not implemented") } +func (*UnimplementedQueueServiceServer) CordonQueue(ctx context.Context, req *QueueCordonRequest) (*types.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CordonQueue not implemented") +} +func (*UnimplementedQueueServiceServer) UncordonQueue(ctx context.Context, req *QueueUncordonRequest) (*types.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method UncordonQueue not implemented") +} func RegisterQueueServiceServer(s *grpc.Server, srv QueueServiceServer) { s.RegisterService(&_QueueService_serviceDesc, srv) @@ -2580,6 +2723,42 @@ func (x *queueServiceGetQueuesServer) Send(m *StreamingQueueMessage) error { return x.ServerStream.SendMsg(m) } +func _QueueService_CordonQueue_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueueCordonRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(QueueServiceServer).CordonQueue(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.QueueService/CordonQueue", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(QueueServiceServer).CordonQueue(ctx, req.(*QueueCordonRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _QueueService_UncordonQueue_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueueUncordonRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(QueueServiceServer).UncordonQueue(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.QueueService/UncordonQueue", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(QueueServiceServer).UncordonQueue(ctx, req.(*QueueUncordonRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _QueueService_serviceDesc = grpc.ServiceDesc{ ServiceName: "api.QueueService", HandlerType: (*QueueServiceServer)(nil), @@ -2608,6 +2787,14 @@ var _QueueService_serviceDesc = grpc.ServiceDesc{ MethodName: "GetQueue", Handler: _QueueService_GetQueue_Handler, }, + { + MethodName: "CordonQueue", + Handler: _QueueService_CordonQueue_Handler, + }, + { + MethodName: "UncordonQueue", + Handler: _QueueService_UncordonQueue_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -4168,6 +4355,25 @@ func (m *Queue) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Labels) > 0 { + for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Labels[iNdEx]) + copy(dAtA[i:], m.Labels[iNdEx]) + i = encodeVarintSubmit(dAtA, i, uint64(len(m.Labels[iNdEx]))) + i-- + dAtA[i] = 0x4a + } + } + if m.Cordoned { + i-- + if m.Cordoned { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x40 + } if len(m.ResourceLimitsByPriorityClassName) > 0 { for k := range m.ResourceLimitsByPriorityClassName { v := m.ResourceLimitsByPriorityClassName[k] @@ -4550,6 +4756,66 @@ func (m *QueueGetRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *QueueCordonRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueueCordonRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueueCordonRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintSubmit(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *QueueUncordonRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueueUncordonRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *QueueUncordonRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintSubmit(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *StreamingQueueGetRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5404,6 +5670,15 @@ func (m *Queue) Size() (n int) { n += mapEntrySize + 1 + sovSubmit(uint64(mapEntrySize)) } } + if m.Cordoned { + n += 2 + } + if len(m.Labels) > 0 { + for _, s := range m.Labels { + l = len(s) + n += 1 + l + sovSubmit(uint64(l)) + } + } return n } @@ -5535,6 +5810,32 @@ func (m *QueueGetRequest) Size() (n int) { return n } +func (m *QueueCordonRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovSubmit(uint64(l)) + } + return n +} + +func (m *QueueUncordonRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovSubmit(uint64(l)) + } + return n +} + func (m *StreamingQueueGetRequest) Size() (n int) { if m == nil { return 0 @@ -9631,6 +9932,58 @@ func (m *Queue) Unmarshal(dAtA []byte) error { } m.ResourceLimitsByPriorityClassName[mapkey] = mapvalue iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Cordoned", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSubmit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Cordoned = bool(v != 0) + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSubmit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSubmit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSubmit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSubmit(dAtA[iNdEx:]) @@ -10571,6 +10924,170 @@ func (m *QueueGetRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *QueueCordonRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSubmit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueueCordonRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueueCordonRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSubmit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSubmit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSubmit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSubmit(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSubmit + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueueUncordonRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSubmit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueueUncordonRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueueUncordonRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSubmit + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSubmit + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSubmit + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSubmit(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSubmit + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StreamingQueueGetRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/api/submit.proto b/pkg/api/submit.proto index 8273e984797..493d25e39bb 100644 --- a/pkg/api/submit.proto +++ b/pkg/api/submit.proto @@ -192,6 +192,10 @@ message Queue { // If provided for a priority class, global limits for that priority class do not apply to this queue. map resource_limits_by_priority_class_name = 7; repeated Permissions permissions = 6; + // Determines whether scheduling is enabled for this queue. + bool cordoned = 8; + // A list of Kubernetes-like key-value labels, e.g. armadaproject.io/priority=critical + repeated string labels = 9; } message PriorityClassResourceLimits { @@ -222,6 +226,16 @@ message QueueGetRequest { string name = 1; } +//swagger:model +message QueueCordonRequest { + string name = 1; +} + +//swagger:model +message QueueUncordonRequest { + string name = 1; +} + //swagger:model message StreamingQueueGetRequest { uint32 num = 1; @@ -274,6 +288,8 @@ service QueueService { rpc DeleteQueue (QueueDeleteRequest) returns (google.protobuf.Empty) {} rpc GetQueue (QueueGetRequest) returns (Queue) {} rpc GetQueues (StreamingQueueGetRequest) returns (stream StreamingQueueMessage) {} + rpc CordonQueue (QueueCordonRequest) returns (google.protobuf.Empty) {} + rpc UncordonQueue (QueueUncordonRequest) returns (google.protobuf.Empty) {} } service Submit { diff --git a/pkg/client/queue/cordon.go b/pkg/client/queue/cordon.go new file mode 100644 index 00000000000..ea3d51cffb4 --- /dev/null +++ b/pkg/client/queue/cordon.go @@ -0,0 +1,59 @@ +package queue + +import ( + "fmt" + + "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/pkg/api" + "github.com/armadaproject/armada/pkg/client" +) + +type CordonAPI func(string) error + +func Cordon(getConnectionDetails client.ConnectionDetails) CordonAPI { + return func(queueName string) error { + connectionDetails, err := getConnectionDetails() + if err != nil { + return fmt.Errorf("failed to obtain api connection details: %s", err) + } + conn, err := client.CreateApiConnection(connectionDetails) + if err != nil { + return fmt.Errorf("failed to connect to api because %s", err) + } + defer conn.Close() + + ctx, cancel := common.ContextWithDefaultTimeout() + defer cancel() + + queueClient := api.NewQueueServiceClient(conn) + if _, err = queueClient.CordonQueue(ctx, &api.QueueCordonRequest{Name: queueName}); err != nil { + return err + } + return nil + } +} + +type UncordonAPI func(string) error + +func Uncordon(getConnectionDetails client.ConnectionDetails) UncordonAPI { + return func(queueName string) error { + connectionDetails, err := getConnectionDetails() + if err != nil { + return fmt.Errorf("failed to obtain api connection details: %s", err) + } + conn, err := client.CreateApiConnection(connectionDetails) + if err != nil { + return fmt.Errorf("failed to connect to api because %s", err) + } + defer conn.Close() + + ctx, cancel := common.ContextWithDefaultTimeout() + defer cancel() + + queueClient := api.NewQueueServiceClient(conn) + if _, err = queueClient.UncordonQueue(ctx, &api.QueueUncordonRequest{Name: queueName}); err != nil { + return err + } + return nil + } +} diff --git a/pkg/client/queue/get_all.go b/pkg/client/queue/get_all.go new file mode 100644 index 00000000000..2357115792d --- /dev/null +++ b/pkg/client/queue/get_all.go @@ -0,0 +1,86 @@ +package queue + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/armadaproject/armada/pkg/api" + "github.com/armadaproject/armada/pkg/client" + + log "github.com/sirupsen/logrus" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type GetAllAPI func() ([]*api.Queue, error) + +func GetAll(getConnectionDetails client.ConnectionDetails) GetAllAPI { + return func() ([]*api.Queue, error) { + connectionDetails, err := getConnectionDetails() + if err != nil { + return nil, fmt.Errorf("failed to obtain api connection details: %s", err) + } + conn, err := client.CreateApiConnection(connectionDetails) + if err != nil { + return nil, fmt.Errorf("failed to connect to api because %s", err) + } + defer conn.Close() + + client := api.NewSubmitClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + allQueues := make([]*api.Queue, 0) + queueStream, err := client.GetQueues(ctx, &api.StreamingQueueGetRequest{}) + if err != nil { + log.Error(err) + return nil, err + } + + for { + + msg, e := queueStream.Recv() + if e != nil { + if err, ok := status.FromError(e); ok { + switch err.Code() { + case codes.NotFound: + log.Error(err.Message()) + return nil, e + case codes.PermissionDenied: + log.Error(err.Message()) + return nil, e + } + } + if e == io.EOF { + return nil, e + } + if !isTransportClosingError(e) { + log.Error(e) + } + break + } + + event := msg.Event + switch event.(type) { + case *api.StreamingQueueMessage_Queue: + allQueues = append(allQueues, event.(*api.StreamingQueueMessage_Queue).Queue) + case *api.StreamingQueueMessage_End: + return allQueues, nil + } + } + + return allQueues, nil + } +} + +func isTransportClosingError(e error) bool { + if err, ok := status.FromError(e); ok { + switch err.Code() { + case codes.Unavailable: + return true + } + } + return false +} diff --git a/pkg/client/queue/queue.go b/pkg/client/queue/queue.go index e7bace8740c..385864df256 100644 --- a/pkg/client/queue/queue.go +++ b/pkg/client/queue/queue.go @@ -2,6 +2,7 @@ package queue import ( "fmt" + "strings" armadamaps "github.com/armadaproject/armada/internal/common/maps" "github.com/armadaproject/armada/pkg/api" @@ -12,6 +13,8 @@ type Queue struct { Permissions []Permissions `json:"permissions"` PriorityFactor PriorityFactor `json:"priorityFactor"` ResourceLimitsByPriorityClassName map[string]api.PriorityClassResourceLimits + Cordoned bool `json:"cordoned"` + Labels []string `json:"labels"` } // NewQueue returns new Queue using the in parameter. Error is returned if @@ -46,11 +49,20 @@ func NewQueue(in *api.Queue) (Queue, error) { } } + // Queue labels must be Kubernetes-like key-value labels + for _, label := range in.Labels { + if len(strings.Split(label, "=")) != 2 { + return Queue{}, fmt.Errorf("queue label must be key-value, not %s", label) + } + } + return Queue{ Name: in.Name, PriorityFactor: priorityFactor, Permissions: permissions, ResourceLimitsByPriorityClassName: resourceLimitsByPriorityClassName, + Cordoned: in.Cordoned, + Labels: in.Labels, }, nil } @@ -64,6 +76,8 @@ func (q Queue) ToAPI() *api.Queue { func(p api.PriorityClassResourceLimits) *api.PriorityClassResourceLimits { return &p }), + Cordoned: q.Cordoned, + Labels: q.Labels, } for _, permission := range q.Permissions { rv.Permissions = append(rv.Permissions, permission.ToAPI()) diff --git a/pkg/client/queue/queue_test.go b/pkg/client/queue/queue_test.go index bcad7b610fa..85cdaa00376 100644 --- a/pkg/client/queue/queue_test.go +++ b/pkg/client/queue/queue_test.go @@ -6,11 +6,14 @@ import ( "testing" "testing/quick" + "github.com/stretchr/testify/require" + "github.com/armadaproject/armada/pkg/api" ) func TestQueue(t *testing.T) { - testCase := func(queue1 Queue) bool { + testCaseIgnoreLabels := func(queue1 Queue) bool { + queue1.Labels = nil queue2, err := NewQueue(queue1.ToAPI()) if err != nil { t.Error(err) @@ -20,11 +23,37 @@ func TestQueue(t *testing.T) { return reflect.DeepEqual(queue1, queue2) } - if err := quick.Check(testCase, nil); err != nil { + if err := quick.Check(testCaseIgnoreLabels, nil); err != nil { t.Fatal(err) } } +func TestQueueWithLabels(t *testing.T) { + queue1 := Queue{ + Name: "queue-a", + PriorityFactor: 100, + Permissions: []Permissions{}, + Labels: []string{"armadaproject.io/gpu-category=gang-user", "armadaproject.io/priority=critical"}, + ResourceLimitsByPriorityClassName: make(map[string]api.PriorityClassResourceLimits), + } + queue2, err := NewQueue(queue1.ToAPI()) + if err != nil { + t.Error(err) + } + + require.True(t, reflect.DeepEqual(queue1, queue2)) +} + +func TestQueueWithIncorrectLabels(t *testing.T) { + queue1 := Queue{ + Name: "queue-a", + PriorityFactor: 100, + Labels: []string{"armadaproject.io/not-key-value"}, + } + _, err := NewQueue(queue1.ToAPI()) + require.Error(t, err) +} + func TestQueueMarshalUnmarshal(t *testing.T) { testCase := func(queue1 Queue) bool { var queue2 Queue