From 5abcfe360c5c41dbc1024d32bf9a0edf01d13deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= Date: Fri, 22 Nov 2024 11:23:49 +0000 Subject: [PATCH 1/6] Protos and parser for managed connectors --- proto/gen/rill/runtime/v1/resources.pb.go | 333 ++++++++++-------- .../rill/runtime/v1/resources.pb.validate.go | 309 ++++++++-------- .../gen/rill/runtime/v1/runtime.swagger.yaml | 4 + proto/rill/runtime/v1/resources.proto | 12 +- runtime/compilers/rillv1/parse_connector.go | 33 ++ runtime/compilers/rillv1/parser_test.go | 73 ++++ .../proto/gen/rill/runtime/v1/resources_pb.ts | 98 +++--- .../src/runtime-client/gen/index.schemas.ts | 12 +- 8 files changed, 528 insertions(+), 346 deletions(-) diff --git a/proto/gen/rill/runtime/v1/resources.pb.go b/proto/gen/rill/runtime/v1/resources.pb.go index 700fba627fd..6b25aff2080 100644 --- a/proto/gen/rill/runtime/v1/resources.pb.go +++ b/proto/gen/rill/runtime/v1/resources.pb.go @@ -5864,6 +5864,61 @@ func (x *CharLocation) GetLine() uint32 { return 0 } +type ConnectorV2 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Spec *ConnectorSpec `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` + State *ConnectorState `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"` +} + +func (x *ConnectorV2) Reset() { + *x = ConnectorV2{} + if protoimpl.UnsafeEnabled { + mi := &file_rill_runtime_v1_resources_proto_msgTypes[71] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnectorV2) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectorV2) ProtoMessage() {} + +func (x *ConnectorV2) ProtoReflect() protoreflect.Message { + mi := &file_rill_runtime_v1_resources_proto_msgTypes[71] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectorV2.ProtoReflect.Descriptor instead. +func (*ConnectorV2) Descriptor() ([]byte, []int) { + return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{71} +} + +func (x *ConnectorV2) GetSpec() *ConnectorSpec { + if x != nil { + return x.Spec + } + return nil +} + +func (x *ConnectorV2) GetState() *ConnectorState { + if x != nil { + return x.State + } + return nil +} + type ConnectorSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -5872,6 +5927,8 @@ type ConnectorSpec struct { Driver string `protobuf:"bytes,1,opt,name=driver,proto3" json:"driver,omitempty"` Properties map[string]string `protobuf:"bytes,2,rep,name=properties,proto3" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` TemplatedProperties []string `protobuf:"bytes,4,rep,name=templated_properties,json=templatedProperties,proto3" json:"templated_properties,omitempty"` + Provision bool `protobuf:"varint,5,opt,name=provision,proto3" json:"provision,omitempty"` + ProvisionArgs *structpb.Struct `protobuf:"bytes,6,opt,name=provision_args,json=provisionArgs,proto3" json:"provision_args,omitempty"` // DEPRECATED: properties_from_variables stores properties whose value is a variable. // NOTE : properties_from_variables and properties both should be used to get all properties. PropertiesFromVariables map[string]string `protobuf:"bytes,3,rep,name=properties_from_variables,json=propertiesFromVariables,proto3" json:"properties_from_variables,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -5880,7 +5937,7 @@ type ConnectorSpec struct { func (x *ConnectorSpec) Reset() { *x = ConnectorSpec{} if protoimpl.UnsafeEnabled { - mi := &file_rill_runtime_v1_resources_proto_msgTypes[71] + mi := &file_rill_runtime_v1_resources_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5893,7 +5950,7 @@ func (x *ConnectorSpec) String() string { func (*ConnectorSpec) ProtoMessage() {} func (x *ConnectorSpec) ProtoReflect() protoreflect.Message { - mi := &file_rill_runtime_v1_resources_proto_msgTypes[71] + mi := &file_rill_runtime_v1_resources_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5906,7 +5963,7 @@ func (x *ConnectorSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectorSpec.ProtoReflect.Descriptor instead. func (*ConnectorSpec) Descriptor() ([]byte, []int) { - return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{71} + return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{72} } func (x *ConnectorSpec) GetDriver() string { @@ -5930,6 +5987,20 @@ func (x *ConnectorSpec) GetTemplatedProperties() []string { return nil } +func (x *ConnectorSpec) GetProvision() bool { + if x != nil { + return x.Provision + } + return false +} + +func (x *ConnectorSpec) GetProvisionArgs() *structpb.Struct { + if x != nil { + return x.ProvisionArgs + } + return nil +} + func (x *ConnectorSpec) GetPropertiesFromVariables() map[string]string { if x != nil { return x.PropertiesFromVariables @@ -5948,7 +6019,7 @@ type ConnectorState struct { func (x *ConnectorState) Reset() { *x = ConnectorState{} if protoimpl.UnsafeEnabled { - mi := &file_rill_runtime_v1_resources_proto_msgTypes[72] + mi := &file_rill_runtime_v1_resources_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5961,7 +6032,7 @@ func (x *ConnectorState) String() string { func (*ConnectorState) ProtoMessage() {} func (x *ConnectorState) ProtoReflect() protoreflect.Message { - mi := &file_rill_runtime_v1_resources_proto_msgTypes[72] + mi := &file_rill_runtime_v1_resources_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5974,7 +6045,7 @@ func (x *ConnectorState) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectorState.ProtoReflect.Descriptor instead. func (*ConnectorState) Descriptor() ([]byte, []int) { - return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{72} + return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{73} } func (x *ConnectorState) GetSpecHash() string { @@ -5984,61 +6055,6 @@ func (x *ConnectorState) GetSpecHash() string { return "" } -type ConnectorV2 struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Spec *ConnectorSpec `protobuf:"bytes,1,opt,name=spec,proto3" json:"spec,omitempty"` - State *ConnectorState `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"` -} - -func (x *ConnectorV2) Reset() { - *x = ConnectorV2{} - if protoimpl.UnsafeEnabled { - mi := &file_rill_runtime_v1_resources_proto_msgTypes[73] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ConnectorV2) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ConnectorV2) ProtoMessage() {} - -func (x *ConnectorV2) ProtoReflect() protoreflect.Message { - mi := &file_rill_runtime_v1_resources_proto_msgTypes[73] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ConnectorV2.ProtoReflect.Descriptor instead. -func (*ConnectorV2) Descriptor() ([]byte, []int) { - return file_rill_runtime_v1_resources_proto_rawDescGZIP(), []int{73} -} - -func (x *ConnectorV2) GetSpec() *ConnectorSpec { - if x != nil { - return x.Spec - } - return nil -} - -func (x *ConnectorV2) GetState() *ConnectorState { - if x != nil { - return x.State - } - return nil -} - // Dimensions are columns to filter and group by type MetricsViewSpec_DimensionV2 struct { state protoimpl.MessageState @@ -7678,37 +7694,7 @@ var file_rill_runtime_v1_resources_proto_rawDesc = []byte{ 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x22, 0x0a, 0x0c, 0x43, 0x68, 0x61, 0x72, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xae, 0x03, - 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, - 0x16, 0x0a, 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x12, 0x4e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x72, 0x69, - 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x31, 0x0a, 0x14, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x64, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x77, 0x0a, 0x19, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x76, 0x61, - 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, - 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x56, 0x61, 0x72, 0x69, - 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x17, 0x70, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x4a, 0x0a, 0x1c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x46, 0x72, 0x6f, 0x6d, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2d, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x70, 0x65, 0x63, 0x48, 0x61, 0x73, 0x68, 0x22, 0x78, 0x0a, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x78, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x56, 0x32, 0x12, 0x32, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, @@ -7716,48 +7702,84 @@ var file_rill_runtime_v1_resources_proto_rawDesc = []byte{ 0x12, 0x35, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2a, 0x8a, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, - 0x6e, 0x63, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x0a, 0x1c, 0x52, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x8c, 0x04, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x72, 0x69, + 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, + 0x72, 0x12, 0x4e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x12, 0x31, 0x0a, 0x14, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x13, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x69, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x3e, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x5f, + 0x61, 0x72, 0x67, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x41, 0x72, + 0x67, 0x73, 0x12, 0x77, 0x0a, 0x19, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, + 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x46, 0x72, 0x6f, 0x6d, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x17, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x46, 0x72, + 0x6f, 0x6d, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4a, 0x0a, 0x1c, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x56, 0x61, 0x72, 0x69, + 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2d, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x70, 0x65, 0x63, + 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x70, 0x65, + 0x63, 0x48, 0x61, 0x73, 0x68, 0x2a, 0x8a, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x63, + 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x0a, 0x1c, 0x52, 0x45, 0x43, + 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, - 0x15, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x43, 0x4f, - 0x4e, 0x43, 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, - 0x44, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, - 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, - 0x4e, 0x47, 0x10, 0x03, 0x2a, 0xab, 0x01, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, - 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x27, - 0x0a, 0x23, 0x45, 0x58, 0x50, 0x4c, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, - 0x49, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x45, 0x58, 0x50, 0x4c, 0x4f, + 0x49, 0x44, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, + 0x49, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, + 0x4e, 0x47, 0x10, 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x43, 0x49, 0x4c, + 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, + 0x10, 0x03, 0x2a, 0xab, 0x01, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, 0x43, 0x6f, + 0x6d, 0x70, 0x61, 0x72, 0x69, 0x73, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x27, 0x0a, 0x23, + 0x45, 0x58, 0x50, 0x4c, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, + 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x45, 0x58, 0x50, 0x4c, 0x4f, 0x52, 0x45, + 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, + 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x45, 0x58, 0x50, 0x4c, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x4f, - 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x45, 0x58, 0x50, + 0x44, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x45, 0x58, 0x50, 0x4c, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, 0x4e, 0x5f, - 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x45, - 0x58, 0x50, 0x4c, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x52, 0x49, 0x53, 0x4f, - 0x4e, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x44, 0x49, 0x4d, 0x45, 0x4e, 0x53, 0x49, 0x4f, 0x4e, - 0x10, 0x03, 0x2a, 0x85, 0x01, 0x0a, 0x0f, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x0a, 0x1c, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x41, 0x53, 0x53, 0x45, - 0x52, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x41, 0x53, - 0x53, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x49, 0x4f, 0x4e, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x02, 0x12, 0x1a, - 0x0a, 0x16, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x42, 0xc1, 0x01, 0x0a, 0x13, 0x63, - 0x6f, 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, - 0x76, 0x31, 0x42, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x72, 0x75, - 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x52, 0x58, 0xaa, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x2e, - 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0f, 0x52, 0x69, 0x6c, - 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1b, 0x52, - 0x69, 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, - 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x11, 0x52, 0x69, 0x6c, - 0x6c, 0x3a, 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x44, 0x49, 0x4d, 0x45, 0x4e, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x03, + 0x2a, 0x85, 0x01, 0x0a, 0x0f, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x0a, 0x1c, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x49, 0x4f, + 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, + 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x41, 0x53, 0x53, 0x10, + 0x01, 0x12, 0x19, 0x0a, 0x15, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, + 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x42, 0xc1, 0x01, 0x0a, 0x13, 0x63, 0x6f, 0x6d, + 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, + 0x42, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, + 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x72, 0x75, 0x6e, 0x74, + 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x76, 0x31, + 0xa2, 0x02, 0x03, 0x52, 0x52, 0x58, 0xaa, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x2e, 0x52, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x5c, + 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1b, 0x52, 0x69, 0x6c, + 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x11, 0x52, 0x69, 0x6c, 0x6c, 0x3a, + 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -7852,9 +7874,9 @@ var file_rill_runtime_v1_resources_proto_goTypes = []any{ (*DependencyError)(nil), // 74: rill.runtime.v1.DependencyError (*ExecutionError)(nil), // 75: rill.runtime.v1.ExecutionError (*CharLocation)(nil), // 76: rill.runtime.v1.CharLocation - (*ConnectorSpec)(nil), // 77: rill.runtime.v1.ConnectorSpec - (*ConnectorState)(nil), // 78: rill.runtime.v1.ConnectorState - (*ConnectorV2)(nil), // 79: rill.runtime.v1.ConnectorV2 + (*ConnectorV2)(nil), // 77: rill.runtime.v1.ConnectorV2 + (*ConnectorSpec)(nil), // 78: rill.runtime.v1.ConnectorSpec + (*ConnectorState)(nil), // 79: rill.runtime.v1.ConnectorState (*MetricsViewSpec_DimensionV2)(nil), // 80: rill.runtime.v1.MetricsViewSpec.DimensionV2 (*MetricsViewSpec_DimensionSelector)(nil), // 81: rill.runtime.v1.MetricsViewSpec.DimensionSelector (*MetricsViewSpec_MeasureWindow)(nil), // 82: rill.runtime.v1.MetricsViewSpec.MeasureWindow @@ -7891,7 +7913,7 @@ var file_rill_runtime_v1_resources_proto_depIdxs = []int32{ 60, // 13: rill.runtime.v1.Resource.component:type_name -> rill.runtime.v1.Component 64, // 14: rill.runtime.v1.Resource.canvas:type_name -> rill.runtime.v1.Canvas 68, // 15: rill.runtime.v1.Resource.api:type_name -> rill.runtime.v1.API - 79, // 16: rill.runtime.v1.Resource.connector:type_name -> rill.runtime.v1.ConnectorV2 + 77, // 16: rill.runtime.v1.Resource.connector:type_name -> rill.runtime.v1.ConnectorV2 8, // 17: rill.runtime.v1.ResourceMeta.name:type_name -> rill.runtime.v1.ResourceName 8, // 18: rill.runtime.v1.ResourceMeta.refs:type_name -> rill.runtime.v1.ResourceName 8, // 19: rill.runtime.v1.ResourceMeta.owner:type_name -> rill.runtime.v1.ResourceName @@ -8016,23 +8038,24 @@ var file_rill_runtime_v1_resources_proto_depIdxs = []int32{ 91, // 138: rill.runtime.v1.APISpec.openapi_parameters:type_name -> google.protobuf.Struct 91, // 139: rill.runtime.v1.APISpec.openapi_response_schema:type_name -> google.protobuf.Struct 76, // 140: rill.runtime.v1.ParseError.start_location:type_name -> rill.runtime.v1.CharLocation - 88, // 141: rill.runtime.v1.ConnectorSpec.properties:type_name -> rill.runtime.v1.ConnectorSpec.PropertiesEntry - 89, // 142: rill.runtime.v1.ConnectorSpec.properties_from_variables:type_name -> rill.runtime.v1.ConnectorSpec.PropertiesFromVariablesEntry - 77, // 143: rill.runtime.v1.ConnectorV2.spec:type_name -> rill.runtime.v1.ConnectorSpec - 78, // 144: rill.runtime.v1.ConnectorV2.state:type_name -> rill.runtime.v1.ConnectorState - 93, // 145: rill.runtime.v1.MetricsViewSpec.DimensionSelector.time_grain:type_name -> rill.runtime.v1.TimeGrain - 81, // 146: rill.runtime.v1.MetricsViewSpec.MeasureWindow.order_by:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionSelector - 3, // 147: rill.runtime.v1.MetricsViewSpec.MeasureV2.type:type_name -> rill.runtime.v1.MetricsViewSpec.MeasureType - 82, // 148: rill.runtime.v1.MetricsViewSpec.MeasureV2.window:type_name -> rill.runtime.v1.MetricsViewSpec.MeasureWindow - 81, // 149: rill.runtime.v1.MetricsViewSpec.MeasureV2.per_dimensions:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionSelector - 81, // 150: rill.runtime.v1.MetricsViewSpec.MeasureV2.required_dimensions:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionSelector - 91, // 151: rill.runtime.v1.MetricsViewSpec.MeasureV2.format_d3_locale:type_name -> google.protobuf.Struct - 84, // 152: rill.runtime.v1.MetricsViewSpec.AvailableTimeRange.comparison_offsets:type_name -> rill.runtime.v1.MetricsViewSpec.AvailableComparisonOffset - 153, // [153:153] is the sub-list for method output_type - 153, // [153:153] is the sub-list for method input_type - 153, // [153:153] is the sub-list for extension type_name - 153, // [153:153] is the sub-list for extension extendee - 0, // [0:153] is the sub-list for field type_name + 78, // 141: rill.runtime.v1.ConnectorV2.spec:type_name -> rill.runtime.v1.ConnectorSpec + 79, // 142: rill.runtime.v1.ConnectorV2.state:type_name -> rill.runtime.v1.ConnectorState + 88, // 143: rill.runtime.v1.ConnectorSpec.properties:type_name -> rill.runtime.v1.ConnectorSpec.PropertiesEntry + 91, // 144: rill.runtime.v1.ConnectorSpec.provision_args:type_name -> google.protobuf.Struct + 89, // 145: rill.runtime.v1.ConnectorSpec.properties_from_variables:type_name -> rill.runtime.v1.ConnectorSpec.PropertiesFromVariablesEntry + 93, // 146: rill.runtime.v1.MetricsViewSpec.DimensionSelector.time_grain:type_name -> rill.runtime.v1.TimeGrain + 81, // 147: rill.runtime.v1.MetricsViewSpec.MeasureWindow.order_by:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionSelector + 3, // 148: rill.runtime.v1.MetricsViewSpec.MeasureV2.type:type_name -> rill.runtime.v1.MetricsViewSpec.MeasureType + 82, // 149: rill.runtime.v1.MetricsViewSpec.MeasureV2.window:type_name -> rill.runtime.v1.MetricsViewSpec.MeasureWindow + 81, // 150: rill.runtime.v1.MetricsViewSpec.MeasureV2.per_dimensions:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionSelector + 81, // 151: rill.runtime.v1.MetricsViewSpec.MeasureV2.required_dimensions:type_name -> rill.runtime.v1.MetricsViewSpec.DimensionSelector + 91, // 152: rill.runtime.v1.MetricsViewSpec.MeasureV2.format_d3_locale:type_name -> google.protobuf.Struct + 84, // 153: rill.runtime.v1.MetricsViewSpec.AvailableTimeRange.comparison_offsets:type_name -> rill.runtime.v1.MetricsViewSpec.AvailableComparisonOffset + 154, // [154:154] is the sub-list for method output_type + 154, // [154:154] is the sub-list for method input_type + 154, // [154:154] is the sub-list for extension type_name + 154, // [154:154] is the sub-list for extension extendee + 0, // [0:154] is the sub-list for field type_name } func init() { file_rill_runtime_v1_resources_proto_init() } @@ -8899,7 +8922,7 @@ func file_rill_runtime_v1_resources_proto_init() { } } file_rill_runtime_v1_resources_proto_msgTypes[71].Exporter = func(v any, i int) any { - switch v := v.(*ConnectorSpec); i { + switch v := v.(*ConnectorV2); i { case 0: return &v.state case 1: @@ -8911,7 +8934,7 @@ func file_rill_runtime_v1_resources_proto_init() { } } file_rill_runtime_v1_resources_proto_msgTypes[72].Exporter = func(v any, i int) any { - switch v := v.(*ConnectorState); i { + switch v := v.(*ConnectorSpec); i { case 0: return &v.state case 1: @@ -8923,7 +8946,7 @@ func file_rill_runtime_v1_resources_proto_init() { } } file_rill_runtime_v1_resources_proto_msgTypes[73].Exporter = func(v any, i int) any { - switch v := v.(*ConnectorV2); i { + switch v := v.(*ConnectorState); i { case 0: return &v.state case 1: diff --git a/proto/gen/rill/runtime/v1/resources.pb.validate.go b/proto/gen/rill/runtime/v1/resources.pb.validate.go index 0b3dfd330d1..0fd216abf18 100644 --- a/proto/gen/rill/runtime/v1/resources.pb.validate.go +++ b/proto/gen/rill/runtime/v1/resources.pb.validate.go @@ -11692,48 +11692,99 @@ var _ interface { ErrorName() string } = CharLocationValidationError{} -// Validate checks the field values on ConnectorSpec with the rules defined in +// Validate checks the field values on ConnectorV2 with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. -func (m *ConnectorSpec) Validate() error { +func (m *ConnectorV2) Validate() error { return m.validate(false) } -// ValidateAll checks the field values on ConnectorSpec with the rules defined -// in the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in ConnectorSpecMultiError, or +// ValidateAll checks the field values on ConnectorV2 with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in ConnectorV2MultiError, or // nil if none found. -func (m *ConnectorSpec) ValidateAll() error { +func (m *ConnectorV2) ValidateAll() error { return m.validate(true) } -func (m *ConnectorSpec) validate(all bool) error { +func (m *ConnectorV2) validate(all bool) error { if m == nil { return nil } var errors []error - // no validation rules for Driver - - // no validation rules for Properties + if all { + switch v := interface{}(m.GetSpec()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ConnectorV2ValidationError{ + field: "Spec", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ConnectorV2ValidationError{ + field: "Spec", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetSpec()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ConnectorV2ValidationError{ + field: "Spec", + reason: "embedded message failed validation", + cause: err, + } + } + } - // no validation rules for PropertiesFromVariables + if all { + switch v := interface{}(m.GetState()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ConnectorV2ValidationError{ + field: "State", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ConnectorV2ValidationError{ + field: "State", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetState()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ConnectorV2ValidationError{ + field: "State", + reason: "embedded message failed validation", + cause: err, + } + } + } if len(errors) > 0 { - return ConnectorSpecMultiError(errors) + return ConnectorV2MultiError(errors) } return nil } -// ConnectorSpecMultiError is an error wrapping multiple validation errors -// returned by ConnectorSpec.ValidateAll() if the designated constraints -// aren't met. -type ConnectorSpecMultiError []error +// ConnectorV2MultiError is an error wrapping multiple validation errors +// returned by ConnectorV2.ValidateAll() if the designated constraints aren't met. +type ConnectorV2MultiError []error // Error returns a concatenation of all the error messages it wraps. -func (m ConnectorSpecMultiError) Error() string { +func (m ConnectorV2MultiError) Error() string { var msgs []string for _, err := range m { msgs = append(msgs, err.Error()) @@ -11742,11 +11793,11 @@ func (m ConnectorSpecMultiError) Error() string { } // AllErrors returns a list of validation violation errors. -func (m ConnectorSpecMultiError) AllErrors() []error { return m } +func (m ConnectorV2MultiError) AllErrors() []error { return m } -// ConnectorSpecValidationError is the validation error returned by -// ConnectorSpec.Validate if the designated constraints aren't met. -type ConnectorSpecValidationError struct { +// ConnectorV2ValidationError is the validation error returned by +// ConnectorV2.Validate if the designated constraints aren't met. +type ConnectorV2ValidationError struct { field string reason string cause error @@ -11754,22 +11805,22 @@ type ConnectorSpecValidationError struct { } // Field function returns field value. -func (e ConnectorSpecValidationError) Field() string { return e.field } +func (e ConnectorV2ValidationError) Field() string { return e.field } // Reason function returns reason value. -func (e ConnectorSpecValidationError) Reason() string { return e.reason } +func (e ConnectorV2ValidationError) Reason() string { return e.reason } // Cause function returns cause value. -func (e ConnectorSpecValidationError) Cause() error { return e.cause } +func (e ConnectorV2ValidationError) Cause() error { return e.cause } // Key function returns key value. -func (e ConnectorSpecValidationError) Key() bool { return e.key } +func (e ConnectorV2ValidationError) Key() bool { return e.key } // ErrorName returns error name. -func (e ConnectorSpecValidationError) ErrorName() string { return "ConnectorSpecValidationError" } +func (e ConnectorV2ValidationError) ErrorName() string { return "ConnectorV2ValidationError" } // Error satisfies the builtin error interface -func (e ConnectorSpecValidationError) Error() string { +func (e ConnectorV2ValidationError) Error() string { cause := "" if e.cause != nil { cause = fmt.Sprintf(" | caused by: %v", e.cause) @@ -11781,14 +11832,14 @@ func (e ConnectorSpecValidationError) Error() string { } return fmt.Sprintf( - "invalid %sConnectorSpec.%s: %s%s", + "invalid %sConnectorV2.%s: %s%s", key, e.field, e.reason, cause) } -var _ error = ConnectorSpecValidationError{} +var _ error = ConnectorV2ValidationError{} var _ interface { Field() string @@ -11796,46 +11847,81 @@ var _ interface { Key() bool Cause() error ErrorName() string -} = ConnectorSpecValidationError{} +} = ConnectorV2ValidationError{} -// Validate checks the field values on ConnectorState with the rules defined in +// Validate checks the field values on ConnectorSpec with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. -func (m *ConnectorState) Validate() error { +func (m *ConnectorSpec) Validate() error { return m.validate(false) } -// ValidateAll checks the field values on ConnectorState with the rules defined +// ValidateAll checks the field values on ConnectorSpec with the rules defined // in the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in ConnectorStateMultiError, -// or nil if none found. -func (m *ConnectorState) ValidateAll() error { +// result is a list of violation errors wrapped in ConnectorSpecMultiError, or +// nil if none found. +func (m *ConnectorSpec) ValidateAll() error { return m.validate(true) } -func (m *ConnectorState) validate(all bool) error { +func (m *ConnectorSpec) validate(all bool) error { if m == nil { return nil } var errors []error - // no validation rules for SpecHash + // no validation rules for Driver + + // no validation rules for Properties + + // no validation rules for Provision + + if all { + switch v := interface{}(m.GetProvisionArgs()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ConnectorSpecValidationError{ + field: "ProvisionArgs", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ConnectorSpecValidationError{ + field: "ProvisionArgs", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetProvisionArgs()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ConnectorSpecValidationError{ + field: "ProvisionArgs", + reason: "embedded message failed validation", + cause: err, + } + } + } + + // no validation rules for PropertiesFromVariables if len(errors) > 0 { - return ConnectorStateMultiError(errors) + return ConnectorSpecMultiError(errors) } return nil } -// ConnectorStateMultiError is an error wrapping multiple validation errors -// returned by ConnectorState.ValidateAll() if the designated constraints +// ConnectorSpecMultiError is an error wrapping multiple validation errors +// returned by ConnectorSpec.ValidateAll() if the designated constraints // aren't met. -type ConnectorStateMultiError []error +type ConnectorSpecMultiError []error // Error returns a concatenation of all the error messages it wraps. -func (m ConnectorStateMultiError) Error() string { +func (m ConnectorSpecMultiError) Error() string { var msgs []string for _, err := range m { msgs = append(msgs, err.Error()) @@ -11844,11 +11930,11 @@ func (m ConnectorStateMultiError) Error() string { } // AllErrors returns a list of validation violation errors. -func (m ConnectorStateMultiError) AllErrors() []error { return m } +func (m ConnectorSpecMultiError) AllErrors() []error { return m } -// ConnectorStateValidationError is the validation error returned by -// ConnectorState.Validate if the designated constraints aren't met. -type ConnectorStateValidationError struct { +// ConnectorSpecValidationError is the validation error returned by +// ConnectorSpec.Validate if the designated constraints aren't met. +type ConnectorSpecValidationError struct { field string reason string cause error @@ -11856,22 +11942,22 @@ type ConnectorStateValidationError struct { } // Field function returns field value. -func (e ConnectorStateValidationError) Field() string { return e.field } +func (e ConnectorSpecValidationError) Field() string { return e.field } // Reason function returns reason value. -func (e ConnectorStateValidationError) Reason() string { return e.reason } +func (e ConnectorSpecValidationError) Reason() string { return e.reason } // Cause function returns cause value. -func (e ConnectorStateValidationError) Cause() error { return e.cause } +func (e ConnectorSpecValidationError) Cause() error { return e.cause } // Key function returns key value. -func (e ConnectorStateValidationError) Key() bool { return e.key } +func (e ConnectorSpecValidationError) Key() bool { return e.key } // ErrorName returns error name. -func (e ConnectorStateValidationError) ErrorName() string { return "ConnectorStateValidationError" } +func (e ConnectorSpecValidationError) ErrorName() string { return "ConnectorSpecValidationError" } // Error satisfies the builtin error interface -func (e ConnectorStateValidationError) Error() string { +func (e ConnectorSpecValidationError) Error() string { cause := "" if e.cause != nil { cause = fmt.Sprintf(" | caused by: %v", e.cause) @@ -11883,14 +11969,14 @@ func (e ConnectorStateValidationError) Error() string { } return fmt.Sprintf( - "invalid %sConnectorState.%s: %s%s", + "invalid %sConnectorSpec.%s: %s%s", key, e.field, e.reason, cause) } -var _ error = ConnectorStateValidationError{} +var _ error = ConnectorSpecValidationError{} var _ interface { Field() string @@ -11898,101 +11984,46 @@ var _ interface { Key() bool Cause() error ErrorName() string -} = ConnectorStateValidationError{} +} = ConnectorSpecValidationError{} -// Validate checks the field values on ConnectorV2 with the rules defined in +// Validate checks the field values on ConnectorState with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. -func (m *ConnectorV2) Validate() error { +func (m *ConnectorState) Validate() error { return m.validate(false) } -// ValidateAll checks the field values on ConnectorV2 with the rules defined in -// the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in ConnectorV2MultiError, or -// nil if none found. -func (m *ConnectorV2) ValidateAll() error { +// ValidateAll checks the field values on ConnectorState with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in ConnectorStateMultiError, +// or nil if none found. +func (m *ConnectorState) ValidateAll() error { return m.validate(true) } -func (m *ConnectorV2) validate(all bool) error { +func (m *ConnectorState) validate(all bool) error { if m == nil { return nil } var errors []error - if all { - switch v := interface{}(m.GetSpec()).(type) { - case interface{ ValidateAll() error }: - if err := v.ValidateAll(); err != nil { - errors = append(errors, ConnectorV2ValidationError{ - field: "Spec", - reason: "embedded message failed validation", - cause: err, - }) - } - case interface{ Validate() error }: - if err := v.Validate(); err != nil { - errors = append(errors, ConnectorV2ValidationError{ - field: "Spec", - reason: "embedded message failed validation", - cause: err, - }) - } - } - } else if v, ok := interface{}(m.GetSpec()).(interface{ Validate() error }); ok { - if err := v.Validate(); err != nil { - return ConnectorV2ValidationError{ - field: "Spec", - reason: "embedded message failed validation", - cause: err, - } - } - } - - if all { - switch v := interface{}(m.GetState()).(type) { - case interface{ ValidateAll() error }: - if err := v.ValidateAll(); err != nil { - errors = append(errors, ConnectorV2ValidationError{ - field: "State", - reason: "embedded message failed validation", - cause: err, - }) - } - case interface{ Validate() error }: - if err := v.Validate(); err != nil { - errors = append(errors, ConnectorV2ValidationError{ - field: "State", - reason: "embedded message failed validation", - cause: err, - }) - } - } - } else if v, ok := interface{}(m.GetState()).(interface{ Validate() error }); ok { - if err := v.Validate(); err != nil { - return ConnectorV2ValidationError{ - field: "State", - reason: "embedded message failed validation", - cause: err, - } - } - } + // no validation rules for SpecHash if len(errors) > 0 { - return ConnectorV2MultiError(errors) + return ConnectorStateMultiError(errors) } return nil } -// ConnectorV2MultiError is an error wrapping multiple validation errors -// returned by ConnectorV2.ValidateAll() if the designated constraints aren't met. -type ConnectorV2MultiError []error +// ConnectorStateMultiError is an error wrapping multiple validation errors +// returned by ConnectorState.ValidateAll() if the designated constraints +// aren't met. +type ConnectorStateMultiError []error // Error returns a concatenation of all the error messages it wraps. -func (m ConnectorV2MultiError) Error() string { +func (m ConnectorStateMultiError) Error() string { var msgs []string for _, err := range m { msgs = append(msgs, err.Error()) @@ -12001,11 +12032,11 @@ func (m ConnectorV2MultiError) Error() string { } // AllErrors returns a list of validation violation errors. -func (m ConnectorV2MultiError) AllErrors() []error { return m } +func (m ConnectorStateMultiError) AllErrors() []error { return m } -// ConnectorV2ValidationError is the validation error returned by -// ConnectorV2.Validate if the designated constraints aren't met. -type ConnectorV2ValidationError struct { +// ConnectorStateValidationError is the validation error returned by +// ConnectorState.Validate if the designated constraints aren't met. +type ConnectorStateValidationError struct { field string reason string cause error @@ -12013,22 +12044,22 @@ type ConnectorV2ValidationError struct { } // Field function returns field value. -func (e ConnectorV2ValidationError) Field() string { return e.field } +func (e ConnectorStateValidationError) Field() string { return e.field } // Reason function returns reason value. -func (e ConnectorV2ValidationError) Reason() string { return e.reason } +func (e ConnectorStateValidationError) Reason() string { return e.reason } // Cause function returns cause value. -func (e ConnectorV2ValidationError) Cause() error { return e.cause } +func (e ConnectorStateValidationError) Cause() error { return e.cause } // Key function returns key value. -func (e ConnectorV2ValidationError) Key() bool { return e.key } +func (e ConnectorStateValidationError) Key() bool { return e.key } // ErrorName returns error name. -func (e ConnectorV2ValidationError) ErrorName() string { return "ConnectorV2ValidationError" } +func (e ConnectorStateValidationError) ErrorName() string { return "ConnectorStateValidationError" } // Error satisfies the builtin error interface -func (e ConnectorV2ValidationError) Error() string { +func (e ConnectorStateValidationError) Error() string { cause := "" if e.cause != nil { cause = fmt.Sprintf(" | caused by: %v", e.cause) @@ -12040,14 +12071,14 @@ func (e ConnectorV2ValidationError) Error() string { } return fmt.Sprintf( - "invalid %sConnectorV2.%s: %s%s", + "invalid %sConnectorState.%s: %s%s", key, e.field, e.reason, cause) } -var _ error = ConnectorV2ValidationError{} +var _ error = ConnectorStateValidationError{} var _ interface { Field() string @@ -12055,7 +12086,7 @@ var _ interface { Key() bool Cause() error ErrorName() string -} = ConnectorV2ValidationError{} +} = ConnectorStateValidationError{} // Validate checks the field values on MetricsViewSpec_DimensionV2 with the // rules defined in the proto definition for this message. If any rules are diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml index 592c1abe7b8..e35f801d2b8 100644 --- a/proto/gen/rill/runtime/v1/runtime.swagger.yaml +++ b/proto/gen/rill/runtime/v1/runtime.swagger.yaml @@ -3953,6 +3953,10 @@ definitions: type: array items: type: string + provision: + type: boolean + provisionArgs: + type: object propertiesFromVariables: type: object additionalProperties: diff --git a/proto/rill/runtime/v1/resources.proto b/proto/rill/runtime/v1/resources.proto index 0760b25633b..49768dc91bd 100644 --- a/proto/rill/runtime/v1/resources.proto +++ b/proto/rill/runtime/v1/resources.proto @@ -726,10 +726,17 @@ message CharLocation { uint32 line = 1; } +message ConnectorV2 { + ConnectorSpec spec = 1; + ConnectorState state = 2; +} + message ConnectorSpec { string driver = 1; map properties = 2; repeated string templated_properties = 4; + bool provision = 5; + google.protobuf.Struct provision_args = 6; // DEPRECATED: properties_from_variables stores properties whose value is a variable. // NOTE : properties_from_variables and properties both should be used to get all properties. map properties_from_variables = 3; @@ -738,8 +745,3 @@ message ConnectorSpec { message ConnectorState { string spec_hash = 1; } - -message ConnectorV2 { - ConnectorSpec spec = 1; - ConnectorState state = 2; -} diff --git a/runtime/compilers/rillv1/parse_connector.go b/runtime/compilers/rillv1/parse_connector.go index b53db1ede42..e98d2100038 100644 --- a/runtime/compilers/rillv1/parse_connector.go +++ b/runtime/compilers/rillv1/parse_connector.go @@ -2,6 +2,9 @@ package rillv1 import ( "fmt" + + "google.golang.org/protobuf/types/known/structpb" + "gopkg.in/yaml.v3" ) // ConnectorYAML is the raw structure of a Connector resource defined in YAML (does not include common fields) @@ -9,6 +12,7 @@ type ConnectorYAML struct { commonYAML `yaml:",inline" mapstructure:",squash"` // Only to avoid loading common fields into Properties // Driver name Driver string `yaml:"driver"` + Managed yaml.Node `yaml:"managed"` // Boolean or map of properties Defaults map[string]string `yaml:",inline" mapstructure:",remain"` } @@ -21,6 +25,33 @@ func (p *Parser) parseConnector(node *Node) error { return err } + // "Managed" indicates that we should automatically provision the connector + var provision bool + var provisionArgsPB *structpb.Struct + if !tmp.Managed.IsZero() { + switch tmp.Managed.Kind { + case yaml.ScalarNode: + err := tmp.Managed.Decode(&provision) + if err != nil { + return fmt.Errorf("failed to decode 'managed': %w", err) + } + case yaml.MappingNode: + provision = true + var provisionArgs map[string]any + err := tmp.Managed.Decode(&provisionArgs) + if err != nil { + return fmt.Errorf("failed to decode 'managed': %w", err) + } + provisionArgsPB, err = structpb.NewStruct(provisionArgs) + if err != nil { + return fmt.Errorf("failed to convert provision args to proto: %w", err) + } + default: + return fmt.Errorf("invalid type for 'managed': expected boolean or map of args") + } + } + + // Find out if any properties are templated templatedProps, err := analyzeTemplatedProperties(tmp.Defaults) if err != nil { return fmt.Errorf("failed to analyze templated properties: %w", err) @@ -36,6 +67,8 @@ func (p *Parser) parseConnector(node *Node) error { r.ConnectorSpec.Driver = tmp.Driver r.ConnectorSpec.Properties = tmp.Defaults r.ConnectorSpec.TemplatedProperties = templatedProps + r.ConnectorSpec.Provision = provision + r.ConnectorSpec.ProvisionArgs = provisionArgsPB return nil } diff --git a/runtime/compilers/rillv1/parser_test.go b/runtime/compilers/rillv1/parser_test.go index 7c616c07606..58d9c8f8f77 100644 --- a/runtime/compilers/rillv1/parser_test.go +++ b/runtime/compilers/rillv1/parser_test.go @@ -2003,6 +2003,79 @@ refresh: requireResourcesAndErrors(t, p, []*Resource{m1, m2}, nil) } +func TestConnector(t *testing.T) { + ctx := context.Background() + repo := makeRepo(t, map[string]string{`rill.yaml`: ``}) + + putRepo(t, repo, map[string]string{ + `connectors/clickhouse.yaml`: ` +type: connector +driver: clickhouse +`}) + r := &Resource{ + Name: ResourceName{Kind: ResourceKindConnector, Name: "clickhouse"}, + Paths: []string{"/connectors/clickhouse.yaml"}, + ConnectorSpec: &runtimev1.ConnectorSpec{ + Driver: "clickhouse", + }, + } + p, err := Parse(ctx, repo, "", "", "duckdb") + require.NoError(t, err) + requireResourcesAndErrors(t, p, []*Resource{r}, nil) + + putRepo(t, repo, map[string]string{ + `connectors/clickhouse.yaml`: ` +type: connector +driver: clickhouse +managed: true +`}) + r = &Resource{ + Name: ResourceName{Kind: ResourceKindConnector, Name: "clickhouse"}, + Paths: []string{"/connectors/clickhouse.yaml"}, + ConnectorSpec: &runtimev1.ConnectorSpec{ + Driver: "clickhouse", + Provision: true, + }, + } + p, err = Parse(ctx, repo, "", "", "duckdb") + require.NoError(t, err) + requireResourcesAndErrors(t, p, []*Resource{r}, nil) + + putRepo(t, repo, map[string]string{ + `connectors/clickhouse.yaml`: ` +type: connector +driver: clickhouse +managed: + hello: world +time_zone: America/Los_Angeles +`}) + r = &Resource{ + Name: ResourceName{Kind: ResourceKindConnector, Name: "clickhouse"}, + Paths: []string{"/connectors/clickhouse.yaml"}, + ConnectorSpec: &runtimev1.ConnectorSpec{ + Driver: "clickhouse", + Properties: map[string]string{"time_zone": "America/Los_Angeles"}, + Provision: true, + ProvisionArgs: must(structpb.NewStruct(map[string]any{"hello": "world"})), + }, + } + p, err = Parse(ctx, repo, "", "", "duckdb") + require.NoError(t, err) + requireResourcesAndErrors(t, p, []*Resource{r}, nil) + + putRepo(t, repo, map[string]string{ + `connectors/clickhouse.yaml`: ` +type: connector +driver: clickhouse +managed: 10 +`}) + p, err = Parse(ctx, repo, "", "", "duckdb") + require.NoError(t, err) + requireResourcesAndErrors(t, p, nil, []*runtimev1.ParseError{ + {Message: "failed to decode 'managed'", FilePath: "/connectors/clickhouse.yaml"}, + }) +} + func requireResourcesAndErrors(t testing.TB, p *Parser, wantResources []*Resource, wantErrors []*runtimev1.ParseError) { // Check errors // NOTE: Assumes there's at most one parse error per file path diff --git a/web-common/src/proto/gen/rill/runtime/v1/resources_pb.ts b/web-common/src/proto/gen/rill/runtime/v1/resources_pb.ts index 63dac18bf7b..8e1e9bdcb74 100644 --- a/web-common/src/proto/gen/rill/runtime/v1/resources_pb.ts +++ b/web-common/src/proto/gen/rill/runtime/v1/resources_pb.ts @@ -4871,6 +4871,49 @@ export class CharLocation extends Message { } } +/** + * @generated from message rill.runtime.v1.ConnectorV2 + */ +export class ConnectorV2 extends Message { + /** + * @generated from field: rill.runtime.v1.ConnectorSpec spec = 1; + */ + spec?: ConnectorSpec; + + /** + * @generated from field: rill.runtime.v1.ConnectorState state = 2; + */ + state?: ConnectorState; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "rill.runtime.v1.ConnectorV2"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "spec", kind: "message", T: ConnectorSpec }, + { no: 2, name: "state", kind: "message", T: ConnectorState }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): ConnectorV2 { + return new ConnectorV2().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): ConnectorV2 { + return new ConnectorV2().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): ConnectorV2 { + return new ConnectorV2().fromJsonString(jsonString, options); + } + + static equals(a: ConnectorV2 | PlainMessage | undefined, b: ConnectorV2 | PlainMessage | undefined): boolean { + return proto3.util.equals(ConnectorV2, a, b); + } +} + /** * @generated from message rill.runtime.v1.ConnectorSpec */ @@ -4890,6 +4933,16 @@ export class ConnectorSpec extends Message { */ templatedProperties: string[] = []; + /** + * @generated from field: bool provision = 5; + */ + provision = false; + + /** + * @generated from field: google.protobuf.Struct provision_args = 6; + */ + provisionArgs?: Struct; + /** * DEPRECATED: properties_from_variables stores properties whose value is a variable. * NOTE : properties_from_variables and properties both should be used to get all properties. @@ -4909,6 +4962,8 @@ export class ConnectorSpec extends Message { { no: 1, name: "driver", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "properties", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "scalar", T: 9 /* ScalarType.STRING */} }, { no: 4, name: "templated_properties", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, + { no: 5, name: "provision", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 6, name: "provision_args", kind: "message", T: Struct }, { no: 3, name: "properties_from_variables", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "scalar", T: 9 /* ScalarType.STRING */} }, ]); @@ -4966,46 +5021,3 @@ export class ConnectorState extends Message { } } -/** - * @generated from message rill.runtime.v1.ConnectorV2 - */ -export class ConnectorV2 extends Message { - /** - * @generated from field: rill.runtime.v1.ConnectorSpec spec = 1; - */ - spec?: ConnectorSpec; - - /** - * @generated from field: rill.runtime.v1.ConnectorState state = 2; - */ - state?: ConnectorState; - - constructor(data?: PartialMessage) { - super(); - proto3.util.initPartial(data, this); - } - - static readonly runtime: typeof proto3 = proto3; - static readonly typeName = "rill.runtime.v1.ConnectorV2"; - static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "spec", kind: "message", T: ConnectorSpec }, - { no: 2, name: "state", kind: "message", T: ConnectorState }, - ]); - - static fromBinary(bytes: Uint8Array, options?: Partial): ConnectorV2 { - return new ConnectorV2().fromBinary(bytes, options); - } - - static fromJson(jsonValue: JsonValue, options?: Partial): ConnectorV2 { - return new ConnectorV2().fromJson(jsonValue, options); - } - - static fromJsonString(jsonString: string, options?: Partial): ConnectorV2 { - return new ConnectorV2().fromJsonString(jsonString, options); - } - - static equals(a: ConnectorV2 | PlainMessage | undefined, b: ConnectorV2 | PlainMessage | undefined): boolean { - return proto3.util.equals(ConnectorV2, a, b); - } -} - diff --git a/web-common/src/runtime-client/gen/index.schemas.ts b/web-common/src/runtime-client/gen/index.schemas.ts index ae008fc6213..7c667c966a1 100644 --- a/web-common/src/runtime-client/gen/index.schemas.ts +++ b/web-common/src/runtime-client/gen/index.schemas.ts @@ -1353,10 +1353,6 @@ export interface V1MetricsViewSort { ascending?: boolean; } -export interface V1MetricsViewSearchResponse { - results?: MetricsViewSearchResponseSearchResult[]; -} - export interface V1MetricsViewSchemaResponse { schema?: V1StructType; } @@ -1976,12 +1972,16 @@ NOTE : properties_from_variables and properties both should be used to get all p */ export type V1ConnectorSpecPropertiesFromVariables = { [key: string]: string }; +export type V1ConnectorSpecProvisionArgs = { [key: string]: any }; + export type V1ConnectorSpecProperties = { [key: string]: string }; export interface V1ConnectorSpec { driver?: string; properties?: V1ConnectorSpecProperties; templatedProperties?: string[]; + provision?: boolean; + provisionArgs?: V1ConnectorSpecProvisionArgs; /** DEPRECATED: properties_from_variables stores properties whose value is a variable. NOTE : properties_from_variables and properties both should be used to get all properties. */ propertiesFromVariables?: V1ConnectorSpecPropertiesFromVariables; @@ -2624,6 +2624,10 @@ export interface MetricsViewSearchResponseSearchResult { value?: unknown; } +export interface V1MetricsViewSearchResponse { + results?: MetricsViewSearchResponseSearchResult[]; +} + export interface MetricsViewFilterCond { name?: string; in?: unknown[]; From b3928be496ffbf8a9a9a181cec3dbddb59de1fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= Date: Fri, 22 Nov 2024 18:38:48 +0000 Subject: [PATCH 2/6] Admin APIs for provisioning --- admin/database/database.go | 43 +- admin/database/postgres/migrations/0055.sql | 5 + admin/database/postgres/postgres.go | 9 + admin/deployments.go | 92 +- admin/permissions.go | 190 +- admin/provision.go | 143 + .../clickhousestatic/provisioner.go | 9 +- admin/provisioner/kubernetes/kubernetes.go | 9 +- admin/provisioner/provisioner.go | 8 +- admin/provisioner/resources.go | 8 + admin/provisioner/static/static.go | 9 +- admin/server/projects.go | 1 + admin/server/provision.go | 102 + docs/docs/manage/roles-permissions.md | 36 +- proto/gen/rill/admin/v1/admin.swagger.yaml | 66 + proto/gen/rill/admin/v1/api.pb.go | 9964 +++++++++-------- proto/gen/rill/admin/v1/api.pb.gw.go | 111 + proto/gen/rill/admin/v1/api.pb.validate.go | 438 + proto/gen/rill/admin/v1/api_grpc.pb.go | 42 + proto/gen/rill/runtime/v1/api.pb.go | 2238 ++-- proto/gen/rill/runtime/v1/api.pb.validate.go | 62 + .../gen/rill/runtime/v1/runtime.swagger.yaml | 11 + proto/rill/admin/v1/api.proto | 37 + proto/rill/runtime/v1/api.proto | 7 + runtime/connection_cache.go | 63 +- runtime/connections.go | 28 +- runtime/drivers/admin.go | 1 + runtime/drivers/admin/admin_store.go | 20 + runtime/drivers/clickhouse/clickhouse.go | 4 +- runtime/reconcilers/connector.go | 15 + runtime/runtime.go | 4 + runtime/server/connectors.go | 11 + .../client/gen/admin-service/admin-service.ts | 53 + web-admin/src/client/gen/index.schemas.ts | 44 +- .../src/proto/gen/rill/admin/v1/api_pb.ts | 181 + .../src/proto/gen/rill/runtime/v1/api_pb.ts | 30 + .../src/runtime-client/gen/index.schemas.ts | 82 +- 37 files changed, 7975 insertions(+), 6201 deletions(-) create mode 100644 admin/database/postgres/migrations/0055.sql create mode 100644 admin/provision.go create mode 100644 admin/server/provision.go diff --git a/admin/database/database.go b/admin/database/database.go index cf15a5530ec..257202f1037 100644 --- a/admin/database/database.go +++ b/admin/database/database.go @@ -293,6 +293,7 @@ type DB interface { DeleteProjectVariables(ctx context.Context, projectID, environment string, vars []string) error FindProvisionerResourcesForDeployment(ctx context.Context, deploymentID string) ([]*ProvisionerResource, error) + FindProvisionerResourceByName(ctx context.Context, deploymentID, typ, name string) (*ProvisionerResource, error) InsertProvisionerResource(ctx context.Context, opts *InsertProvisionerResourceOptions) (*ProvisionerResource, error) UpdateProvisionerResource(ctx context.Context, id string, opts *UpdateProvisionerResourceOptions) (*ProvisionerResource, error) DeleteProvisionerResource(ctx context.Context, id string) error @@ -783,26 +784,28 @@ type OrganizationRole struct { // ProjectRole represents roles for projects. type ProjectRole struct { - ID string - Name string - ReadProject bool `db:"read_project"` - ManageProject bool `db:"manage_project"` - ReadProd bool `db:"read_prod"` - ReadProdStatus bool `db:"read_prod_status"` - ManageProd bool `db:"manage_prod"` - ReadDev bool `db:"read_dev"` - ReadDevStatus bool `db:"read_dev_status"` - ManageDev bool `db:"manage_dev"` - ReadProjectMembers bool `db:"read_project_members"` - ManageProjectMembers bool `db:"manage_project_members"` - CreateMagicAuthTokens bool `db:"create_magic_auth_tokens"` - ManageMagicAuthTokens bool `db:"manage_magic_auth_tokens"` - CreateReports bool `db:"create_reports"` - ManageReports bool `db:"manage_reports"` - CreateAlerts bool `db:"create_alerts"` - ManageAlerts bool `db:"manage_alerts"` - CreateBookmarks bool `db:"create_bookmarks"` - ManageBookmarks bool `db:"manage_bookmarks"` + ID string + Name string + ReadProject bool `db:"read_project"` + ManageProject bool `db:"manage_project"` + ReadProd bool `db:"read_prod"` + ReadProdStatus bool `db:"read_prod_status"` + ManageProd bool `db:"manage_prod"` + ReadDev bool `db:"read_dev"` + ReadDevStatus bool `db:"read_dev_status"` + ManageDev bool `db:"manage_dev"` + ReadProvisionerResources bool `db:"read_provisioner_resources"` + ManageProvisionerResources bool `db:"manage_provisioner_resources"` + ReadProjectMembers bool `db:"read_project_members"` + ManageProjectMembers bool `db:"manage_project_members"` + CreateMagicAuthTokens bool `db:"create_magic_auth_tokens"` + ManageMagicAuthTokens bool `db:"manage_magic_auth_tokens"` + CreateReports bool `db:"create_reports"` + ManageReports bool `db:"manage_reports"` + CreateAlerts bool `db:"create_alerts"` + ManageAlerts bool `db:"manage_alerts"` + CreateBookmarks bool `db:"create_bookmarks"` + ManageBookmarks bool `db:"manage_bookmarks"` } // MemberUser is a convenience type used for display-friendly representation of an org or project member. diff --git a/admin/database/postgres/migrations/0055.sql b/admin/database/postgres/migrations/0055.sql new file mode 100644 index 00000000000..fe4836ee2eb --- /dev/null +++ b/admin/database/postgres/migrations/0055.sql @@ -0,0 +1,5 @@ +ALTER TABLE project_roles ADD read_provisioner_resources BOOLEAN DEFAULT false NOT NULL; +UPDATE project_roles SET read_provisioner_resources = read_prod_status; + +ALTER TABLE project_roles ADD manage_provisioner_resources BOOLEAN DEFAULT false NOT NULL; +UPDATE project_roles SET manage_provisioner_resources = manage_prod; diff --git a/admin/database/postgres/postgres.go b/admin/database/postgres/postgres.go index f535a56beb1..14da4b9f025 100644 --- a/admin/database/postgres/postgres.go +++ b/admin/database/postgres/postgres.go @@ -2232,6 +2232,15 @@ func (c *connection) FindProvisionerResourcesForDeployment(ctx context.Context, return c.provisionerResourcesFromDTOs(res) } +func (c *connection) FindProvisionerResourceByName(ctx context.Context, deploymentID, typ, name string) (*database.ProvisionerResource, error) { + res := &provisionerResourceDTO{} + err := c.getDB(ctx).QueryRowxContext(ctx, `SELECT * FROM provisioner_resources WHERE deployment_id = $1 AND "type" = $2 AND name = $3`, deploymentID, typ, name).StructScan(res) + if err != nil { + return nil, parseErr("provisioner resource", err) + } + return c.provisionerResourceFromDTO(res) +} + func (c *connection) InsertProvisionerResource(ctx context.Context, opts *database.InsertProvisionerResourceOptions) (*database.ProvisionerResource, error) { if err := database.Validate(opts); err != nil { return nil, err diff --git a/admin/deployments.go b/admin/deployments.go index 6bcd7592151..81181ea7cea 100644 --- a/admin/deployments.go +++ b/admin/deployments.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/google/uuid" "github.com/hashicorp/go-version" "github.com/rilldata/rill/admin/database" "github.com/rilldata/rill/admin/provisioner" @@ -478,15 +477,10 @@ type provisionRuntimeOptions struct { } func (s *Service) provisionRuntime(ctx context.Context, opts *provisionRuntimeOptions) (*database.ProvisionerResource, error) { - // Get provisioner from the set. // Use default if no provisioner is specified. if opts.Provisioner == "" { opts.Provisioner = s.opts.DefaultProvisioner } - p, ok := s.ProvisionerSet[opts.Provisioner] - if !ok { - return nil, fmt.Errorf("provisioner: %q is not in the provisioner set", opts.Provisioner) - } // Create provisioner args args := &provisioner.RuntimeArgs{ @@ -494,91 +488,31 @@ func (s *Service) provisionRuntime(ctx context.Context, opts *provisionRuntimeOp Version: opts.Version, } - // Attempt to find an existing provisioned runtime for the deployment - pr, ok, err := s.findProvisionedRuntimeResource(ctx, opts.DeploymentID) - if err != nil { - return nil, err - } - if ok && pr.Provisioner != opts.Provisioner { - return nil, fmt.Errorf("provisioner: cannot change provisioner from %q to %q for deployment %q", pr.Provisioner, opts.Provisioner, opts.DeploymentID) - } - - // If we didn't find an existing DB entry, create one - if !ok { - pr, err = s.DB.InsertProvisionerResource(ctx, &database.InsertProvisionerResourceOptions{ - ID: uuid.New().String(), - DeploymentID: opts.DeploymentID, - Type: string(provisioner.ResourceTypeRuntime), - Name: "", // Not giving runtime resources a name since there should only be one per deployment. - Status: database.ProvisionerResourceStatusPending, - StatusMessage: "Provisioning...", - Provisioner: opts.Provisioner, - Args: args.AsMap(), - State: nil, // Will be populated after provisioning - Config: nil, // Will be populated after provisioning - }) - if err != nil { - return nil, err - } - } - - // Provision the runtime - r := &provisioner.Resource{ - ID: pr.ID, - Type: provisioner.ResourceTypeRuntime, - State: pr.State, // Empty if inserting - Config: pr.Config, // Empty if inserting - } - r, err = p.Provision(ctx, r, &provisioner.ResourceOptions{ - Args: args.AsMap(), - Annotations: opts.Annotations, - RillVersion: s.resolveRillVersion(), - }) - if err != nil { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - _, _ = s.DB.UpdateProvisionerResource(ctx, pr.ID, &database.UpdateProvisionerResourceOptions{ - Status: database.ProvisionerResourceStatusError, - StatusMessage: fmt.Sprintf("Failed provisioning runtime: %v", err), - Args: pr.Args, - State: pr.State, - Config: pr.Config, - }) - return nil, err - } - - // Update the provisioner resource - pr, err = s.DB.UpdateProvisionerResource(ctx, pr.ID, &database.UpdateProvisionerResourceOptions{ - Status: database.ProvisionerResourceStatusOK, - StatusMessage: "", - Args: args.AsMap(), - State: r.State, - Config: r.Config, + // Call into the generic provision function + pr, err := s.Provision(ctx, &ProvisionOptions{ + DeploymentID: opts.DeploymentID, + Type: provisioner.ResourceTypeRuntime, + Name: "", // Not giving runtime resources a name since there should only be one per deployment. + Provisioner: opts.Provisioner, + Args: args.AsMap(), + Annotations: opts.Annotations, }) if err != nil { return nil, err } - // Await the runtime to be ready - err = p.AwaitReady(ctx, r) - if err != nil { - return nil, err - } - return pr, nil } func (s *Service) findProvisionedRuntimeResource(ctx context.Context, deploymentID string) (*database.ProvisionerResource, bool, error) { - prs, err := s.DB.FindProvisionerResourcesForDeployment(ctx, deploymentID) + pr, err := s.DB.FindProvisionerResourceByName(ctx, deploymentID, string(provisioner.ResourceTypeRuntime), "") if err != nil { - return nil, false, err - } - for _, val := range prs { - if provisioner.ResourceType(val.Type) == provisioner.ResourceTypeRuntime { - return val, true, nil + if errors.Is(err, database.ErrNotFound) { + return nil, false, nil } + return nil, false, err } - return nil, false, nil + return pr, true, nil } func (s *Service) resolveRillVersion() string { diff --git a/admin/permissions.go b/admin/permissions.go index ecbb45fa084..58abca136ce 100644 --- a/admin/permissions.go +++ b/admin/permissions.go @@ -104,24 +104,26 @@ func (s *Service) ProjectPermissionsForUser(ctx context.Context, projectID, user // ManageProjects permission on the org gives full access to all projects in the org (only org admins have this) if orgPerms.ManageProjects { return &adminv1.ProjectPermissions{ - ReadProject: true, - ManageProject: true, - ReadProd: true, - ReadProdStatus: true, - ManageProd: true, - ReadDev: true, - ReadDevStatus: true, - ManageDev: true, - ReadProjectMembers: true, - ManageProjectMembers: true, - CreateMagicAuthTokens: true, - ManageMagicAuthTokens: true, - CreateReports: true, - ManageReports: true, - CreateAlerts: true, - ManageAlerts: true, - CreateBookmarks: true, - ManageBookmarks: true, + ReadProject: true, + ManageProject: true, + ReadProd: true, + ReadProdStatus: true, + ManageProd: true, + ReadDev: true, + ReadDevStatus: true, + ManageDev: true, + ReadProvisionerResources: true, + ManageProvisionerResources: true, + ReadProjectMembers: true, + ManageProjectMembers: true, + CreateMagicAuthTokens: true, + ManageMagicAuthTokens: true, + CreateReports: true, + ManageReports: true, + CreateAlerts: true, + ManageAlerts: true, + CreateBookmarks: true, + ManageBookmarks: true, }, nil } @@ -143,24 +145,26 @@ func (s *Service) ProjectPermissionsForUser(ctx context.Context, projectID, user func (s *Service) ProjectPermissionsForService(ctx context.Context, projectID, serviceID string, orgPerms *adminv1.OrganizationPermissions) (*adminv1.ProjectPermissions, error) { if orgPerms.ManageProjects { return &adminv1.ProjectPermissions{ - ReadProject: true, - ManageProject: true, - ReadProd: true, - ReadProdStatus: true, - ManageProd: true, - ReadDev: true, - ReadDevStatus: true, - ManageDev: true, - ReadProjectMembers: true, - ManageProjectMembers: true, - CreateMagicAuthTokens: true, - ManageMagicAuthTokens: true, - CreateReports: true, - ManageReports: true, - CreateAlerts: true, - ManageAlerts: true, - CreateBookmarks: true, - ManageBookmarks: true, + ReadProject: true, + ManageProject: true, + ReadProd: true, + ReadProdStatus: true, + ManageProd: true, + ReadDev: true, + ReadDevStatus: true, + ManageDev: true, + ReadProvisionerResources: true, + ManageProvisionerResources: true, + ReadProjectMembers: true, + ManageProjectMembers: true, + CreateMagicAuthTokens: true, + ManageMagicAuthTokens: true, + CreateReports: true, + ManageReports: true, + CreateAlerts: true, + ManageAlerts: true, + CreateBookmarks: true, + ManageBookmarks: true, }, nil } @@ -178,24 +182,26 @@ func (s *Service) ProjectPermissionsForDeployment(ctx context.Context, projectID // Deployments get full read and no write permissions on the project they belong to if projectID == depl.ProjectID { return &adminv1.ProjectPermissions{ - ReadProject: true, - ManageProject: false, - ReadProd: true, - ReadProdStatus: true, - ManageProd: false, - ReadDev: true, - ReadDevStatus: true, - ManageDev: false, - ReadProjectMembers: true, - ManageProjectMembers: false, - CreateMagicAuthTokens: false, - ManageMagicAuthTokens: false, - CreateReports: false, - ManageReports: false, - CreateAlerts: false, - ManageAlerts: false, - CreateBookmarks: false, - ManageBookmarks: false, + ReadProject: true, + ManageProject: false, + ReadProd: true, + ReadProdStatus: true, + ManageProd: false, + ReadDev: true, + ReadDevStatus: true, + ManageDev: false, + ReadProvisionerResources: true, + ManageProvisionerResources: true, + ReadProjectMembers: true, + ManageProjectMembers: false, + CreateMagicAuthTokens: false, + ManageMagicAuthTokens: false, + CreateReports: false, + ManageReports: false, + CreateAlerts: false, + ManageAlerts: false, + CreateBookmarks: false, + ManageBookmarks: false, }, nil } @@ -211,24 +217,26 @@ func (s *Service) ProjectPermissionsForMagicAuthToken(ctx context.Context, proje // Grant basic read access to the project and its prod deployment return &adminv1.ProjectPermissions{ - ReadProject: true, - ManageProject: false, - ReadProd: true, - ReadProdStatus: false, - ManageProd: false, - ReadDev: false, - ReadDevStatus: false, - ManageDev: false, - ReadProjectMembers: false, - ManageProjectMembers: false, - CreateMagicAuthTokens: false, - ManageMagicAuthTokens: false, - CreateReports: false, - ManageReports: false, - CreateAlerts: false, - ManageAlerts: false, - CreateBookmarks: false, - ManageBookmarks: false, + ReadProject: true, + ManageProject: false, + ReadProd: true, + ReadProdStatus: false, + ManageProd: false, + ReadDev: false, + ReadDevStatus: false, + ManageDev: false, + ReadProvisionerResources: false, + ManageProvisionerResources: false, + ReadProjectMembers: false, + ManageProjectMembers: false, + CreateMagicAuthTokens: false, + ManageMagicAuthTokens: false, + CreateReports: false, + ManageReports: false, + CreateAlerts: false, + ManageAlerts: false, + CreateBookmarks: false, + ManageBookmarks: false, }, nil } @@ -246,23 +254,25 @@ func unionOrgRoles(a *adminv1.OrganizationPermissions, b *database.OrganizationR func unionProjectRoles(a *adminv1.ProjectPermissions, b *database.ProjectRole) *adminv1.ProjectPermissions { return &adminv1.ProjectPermissions{ - ReadProject: a.ReadProject || b.ReadProject, - ManageProject: a.ManageProject || b.ManageProject, - ReadProd: a.ReadProd || b.ReadProd, - ReadProdStatus: a.ReadProdStatus || b.ReadProdStatus, - ManageProd: a.ManageProd || b.ManageProd, - ReadDev: a.ReadDev || b.ReadDev, - ReadDevStatus: a.ReadDevStatus || b.ReadDevStatus, - ManageDev: a.ManageDev || b.ManageDev, - ReadProjectMembers: a.ReadProjectMembers || b.ReadProjectMembers, - ManageProjectMembers: a.ManageProjectMembers || b.ManageProjectMembers, - CreateMagicAuthTokens: a.CreateMagicAuthTokens || b.CreateMagicAuthTokens, - ManageMagicAuthTokens: a.ManageMagicAuthTokens || b.ManageMagicAuthTokens, - CreateReports: a.CreateReports || b.CreateReports, - ManageReports: a.ManageReports || b.ManageReports, - CreateAlerts: a.CreateAlerts || b.CreateAlerts, - ManageAlerts: a.ManageAlerts || b.ManageAlerts, - CreateBookmarks: a.CreateBookmarks || b.CreateBookmarks, - ManageBookmarks: a.ManageBookmarks || b.ManageBookmarks, + ReadProject: a.ReadProject || b.ReadProject, + ManageProject: a.ManageProject || b.ManageProject, + ReadProd: a.ReadProd || b.ReadProd, + ReadProdStatus: a.ReadProdStatus || b.ReadProdStatus, + ManageProd: a.ManageProd || b.ManageProd, + ReadDev: a.ReadDev || b.ReadDev, + ReadDevStatus: a.ReadDevStatus || b.ReadDevStatus, + ManageDev: a.ManageDev || b.ManageDev, + ReadProvisionerResources: a.ReadProvisionerResources || b.ReadProvisionerResources, + ManageProvisionerResources: a.ManageProvisionerResources || b.ManageProvisionerResources, + ReadProjectMembers: a.ReadProjectMembers || b.ReadProjectMembers, + ManageProjectMembers: a.ManageProjectMembers || b.ManageProjectMembers, + CreateMagicAuthTokens: a.CreateMagicAuthTokens || b.CreateMagicAuthTokens, + ManageMagicAuthTokens: a.ManageMagicAuthTokens || b.ManageMagicAuthTokens, + CreateReports: a.CreateReports || b.CreateReports, + ManageReports: a.ManageReports || b.ManageReports, + CreateAlerts: a.CreateAlerts || b.CreateAlerts, + ManageAlerts: a.ManageAlerts || b.ManageAlerts, + CreateBookmarks: a.CreateBookmarks || b.CreateBookmarks, + ManageBookmarks: a.ManageBookmarks || b.ManageBookmarks, } } diff --git a/admin/provision.go b/admin/provision.go new file mode 100644 index 00000000000..864846faba9 --- /dev/null +++ b/admin/provision.go @@ -0,0 +1,143 @@ +package admin + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/rilldata/rill/admin/database" + "github.com/rilldata/rill/admin/provisioner" +) + +type ProvisionOptions struct { + DeploymentID string + Type provisioner.ResourceType + Name string + Provisioner string + Args map[string]any + Annotations map[string]string +} + +func (s *Service) Provision(ctx context.Context, opts *ProvisionOptions) (*database.ProvisionerResource, error) { + // Attempt to find an existing provisioned resource + pr, err := s.DB.FindProvisionerResourceByName(ctx, opts.DeploymentID, string(opts.Type), opts.Name) + if err != nil && !errors.Is(err, database.ErrNotFound) { + return nil, err + } + + // Find the provisioner to use + var provisionerName string + var p provisioner.Provisioner + if pr != nil { + if opts.Provisioner != "" && opts.Provisioner != pr.Provisioner { + return nil, fmt.Errorf("provisioner: cannot change provisioner from %q to %q for deployment %q", provisionerName, opts.Provisioner, opts.DeploymentID) + } + + var ok bool + provisionerName = pr.Provisioner + p, ok = s.ProvisionerSet[provisionerName] + if !ok { + return nil, fmt.Errorf("provisioner: previous provisioner %q is no longer in the provisioner set", provisionerName) + } + + if !p.Supports(opts.Type) { + return nil, fmt.Errorf("provisioner: previous provisioner %q no longer supports resource type %q", provisionerName, opts.Type) + } + } else if opts.Provisioner != "" { + provisionerName = opts.Provisioner + var ok bool + p, ok = s.ProvisionerSet[provisionerName] + if !ok { + return nil, fmt.Errorf("provisioner: the requested provisioner %q is not in the provisioner set", provisionerName) + } + + if !p.Supports(opts.Type) { + return nil, fmt.Errorf("provisioner: the requested provisioner %q does not support resource type %q", provisionerName, opts.Type) + } + } else { + for n, candidate := range s.ProvisionerSet { + if candidate.Supports(opts.Type) { + provisionerName = n + p = candidate + break + } + } + if p == nil { + return nil, fmt.Errorf("provisioner: no provisioner available that supports resource type %q", opts.Type) + } + } + + // Insert a pending provisioner resource if it doesn't exist + if pr == nil { + pr, err = s.DB.InsertProvisionerResource(ctx, &database.InsertProvisionerResourceOptions{ + ID: uuid.New().String(), + DeploymentID: opts.DeploymentID, + Type: string(opts.Type), + Name: opts.Name, + Status: database.ProvisionerResourceStatusPending, + StatusMessage: "Provisioning...", + Provisioner: provisionerName, + Args: opts.Args, + State: nil, // Will be populated after provisioning + Config: nil, // Will be populated after provisioning + }) + if err != nil { + if !errors.Is(err, database.ErrNotUnique) { + return nil, err + } + + // The resource must have been created concurrently by another process, so we try to find it again. + pr, err = s.DB.FindProvisionerResourceByName(ctx, opts.DeploymentID, string(opts.Type), opts.Name) + if err != nil { + return nil, fmt.Errorf("failed to find expected provisioner resource: %w", err) + } + } + } + + // Provision the resource + r := &provisioner.Resource{ + ID: pr.ID, + Type: opts.Type, + State: pr.State, // Empty if inserting + Config: pr.Config, // Empty if inserting + } + r, err = p.Provision(ctx, r, &provisioner.ResourceOptions{ + Args: opts.Args, + Annotations: opts.Annotations, + RillVersion: s.resolveRillVersion(), + }) + if err != nil { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, _ = s.DB.UpdateProvisionerResource(ctx, pr.ID, &database.UpdateProvisionerResourceOptions{ + Status: database.ProvisionerResourceStatusError, + StatusMessage: fmt.Sprintf("Failed provisioning: %v", err), + Args: pr.Args, + State: pr.State, + Config: pr.Config, + }) + return nil, err + } + + // Update the provisioner resource + pr, err = s.DB.UpdateProvisionerResource(ctx, pr.ID, &database.UpdateProvisionerResourceOptions{ + Status: database.ProvisionerResourceStatusOK, + StatusMessage: "", + Args: opts.Args, + State: r.State, + Config: r.Config, + }) + if err != nil { + return nil, err + } + + // Await the resource to be ready + err = p.AwaitReady(ctx, r) + if err != nil { + return nil, err + } + + return pr, nil +} diff --git a/admin/provisioner/clickhousestatic/provisioner.go b/admin/provisioner/clickhousestatic/provisioner.go index 1432e3c8649..35cdbd3ae0e 100644 --- a/admin/provisioner/clickhousestatic/provisioner.go +++ b/admin/provisioner/clickhousestatic/provisioner.go @@ -60,16 +60,15 @@ func (p *Provisioner) Type() string { return "clickhouse-static" } +func (p *Provisioner) Supports(rt provisioner.ResourceType) bool { + return rt == provisioner.ResourceTypeClickHouse +} + func (p *Provisioner) Close() error { return p.ch.Close() } func (p *Provisioner) Provision(ctx context.Context, r *provisioner.Resource, opts *provisioner.ResourceOptions) (*provisioner.Resource, error) { - // Can only provision clickhouse resources - if r.Type != provisioner.ResourceTypeClickHouse { - return nil, provisioner.ErrResourceTypeNotSupported - } - // Parse the resource's config (in case it's an update/check) cfg, err := provisioner.NewClickhouseConfig(r.Config) if err != nil { diff --git a/admin/provisioner/kubernetes/kubernetes.go b/admin/provisioner/kubernetes/kubernetes.go index f6c350c3f4e..220fa95f17e 100644 --- a/admin/provisioner/kubernetes/kubernetes.go +++ b/admin/provisioner/kubernetes/kubernetes.go @@ -144,16 +144,15 @@ func (p *KubernetesProvisioner) Type() string { return "kubernetes" } +func (p *KubernetesProvisioner) Supports(rt provisioner.ResourceType) bool { + return rt == provisioner.ResourceTypeRuntime +} + func (p *KubernetesProvisioner) Close() error { return nil } func (p *KubernetesProvisioner) Provision(ctx context.Context, r *provisioner.Resource, opts *provisioner.ResourceOptions) (*provisioner.Resource, error) { - // Can only provision runtime resources - if r.Type != provisioner.ResourceTypeRuntime { - return nil, provisioner.ErrResourceTypeNotSupported - } - // Parse args args, err := provisioner.NewRuntimeArgs(opts.Args) if err != nil { diff --git a/admin/provisioner/provisioner.go b/admin/provisioner/provisioner.go index 80a68089e76..c4b8b943961 100644 --- a/admin/provisioner/provisioner.go +++ b/admin/provisioner/provisioner.go @@ -3,18 +3,12 @@ package provisioner import ( "context" "encoding/json" - "errors" "fmt" "github.com/rilldata/rill/admin/database" "go.uber.org/zap" ) -// ErrResourceTypeNotSupported should be returned by Provision if the provisioner does not support the requested resource type. -// -// By checking for this error, we can iterate over the chain of provisioners until we find a provisioner capable of provisioning the requested service. -var ErrResourceTypeNotSupported = errors.New("provisioner: resource type not supported") - // ProvisionerInitializer creates a new provisioner. type ProvisionerInitializer func(specJSON []byte, db database.DB, logger *zap.Logger) (Provisioner, error) @@ -35,6 +29,8 @@ type Provisioner interface { Type() string // Close is called when the provisioner is no longer needed. Close() error + // Supports indicates if it can provision the resource type. + Supports(rt ResourceType) bool // Provision provisions a new resource. // It may be called multiple times for the same ID if: // - the initial provision is interrupted, or diff --git a/admin/provisioner/resources.go b/admin/provisioner/resources.go index ab2ba43b39d..f545473c052 100644 --- a/admin/provisioner/resources.go +++ b/admin/provisioner/resources.go @@ -14,6 +14,14 @@ const ( ResourceTypeClickHouse ResourceType = "clickhouse" ) +func (r ResourceType) Valid() bool { + switch r { + case ResourceTypeRuntime, ResourceTypeClickHouse: + return true + } + return false +} + // RuntimeArgs describe the expected arguments for provisioning a runtime resource. type RuntimeArgs struct { Slots int `mapstructure:"slots"` diff --git a/admin/provisioner/static/static.go b/admin/provisioner/static/static.go index f2d276aabd3..841cd429019 100644 --- a/admin/provisioner/static/static.go +++ b/admin/provisioner/static/static.go @@ -62,16 +62,15 @@ func (p *StaticProvisioner) Type() string { return "static" } +func (p *StaticProvisioner) Supports(rt provisioner.ResourceType) bool { + return rt == provisioner.ResourceTypeRuntime +} + func (p *StaticProvisioner) Close() error { return nil } func (p *StaticProvisioner) Provision(ctx context.Context, r *provisioner.Resource, opts *provisioner.ResourceOptions) (*provisioner.Resource, error) { - // Can only provision runtime resources - if r.Type != provisioner.ResourceTypeRuntime { - return nil, provisioner.ErrResourceTypeNotSupported - } - // Parse args args, err := provisioner.NewRuntimeArgs(opts.Args) if err != nil { diff --git a/admin/server/projects.go b/admin/server/projects.go index abce4a28d74..1d36b477be5 100644 --- a/admin/server/projects.go +++ b/admin/server/projects.go @@ -121,6 +121,7 @@ func (s *Server) GetProject(ctx context.Context, req *adminv1.GetProjectRequest) permissions.ReadProdStatus = true permissions.ReadDev = true permissions.ReadDevStatus = true + permissions.ReadProvisionerResources = true permissions.ReadProjectMembers = true } diff --git a/admin/server/provision.go b/admin/server/provision.go new file mode 100644 index 00000000000..65d679f6166 --- /dev/null +++ b/admin/server/provision.go @@ -0,0 +1,102 @@ +package server + +import ( + "context" + "errors" + + "github.com/rilldata/rill/admin" + "github.com/rilldata/rill/admin/database" + "github.com/rilldata/rill/admin/provisioner" + "github.com/rilldata/rill/admin/server/auth" + adminv1 "github.com/rilldata/rill/proto/gen/rill/admin/v1" + "github.com/rilldata/rill/runtime/pkg/observability" + "go.opentelemetry.io/otel/attribute" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" +) + +func (s *Server) Provision(ctx context.Context, req *adminv1.ProvisionRequest) (*adminv1.ProvisionResponse, error) { + observability.AddRequestAttributes(ctx, + attribute.String("args.deployment_id", req.DeploymentId), + attribute.String("args.type", req.Type), + attribute.String("args.name", req.Name), + ) + + // If the deployment ID is not provided, attempt to infer it from the access token. + claims := auth.GetClaims(ctx) + if req.DeploymentId == "" { + if claims.OwnerType() == auth.OwnerTypeDeployment { + req.DeploymentId = claims.OwnerID() + } else { + return nil, status.Error(codes.InvalidArgument, "missing deployment_id") + } + } + + depl, err := s.admin.DB.FindDeployment(ctx, req.DeploymentId) + if err != nil { + return nil, err + } + + proj, err := s.admin.DB.FindProject(ctx, depl.ProjectID) + if err != nil { + return nil, err + } + + permissions := auth.GetClaims(ctx).ProjectPermissions(ctx, proj.OrganizationID, proj.ID) + if !permissions.ManageProvisionerResources { + return nil, status.Error(codes.PermissionDenied, "not allowed to manage provisioner resources") + } + + // If the resource is OK, return it immediately. + res, err := s.admin.DB.FindProvisionerResourceByName(ctx, depl.ID, req.Type, req.Name) + if err != nil && !errors.Is(err, database.ErrNotFound) { + return nil, err + } + if res != nil && res.Status == database.ProvisionerResourceStatusOK { + return &adminv1.ProvisionResponse{ + Resource: provisionerResourceToPB(res), + }, nil + } + + // Try or retry provisioning the resource. + org, err := s.admin.DB.FindOrganization(ctx, proj.OrganizationID) + if err != nil { + return nil, err + } + typ := provisioner.ResourceType(req.Type) + if !typ.Valid() { + return nil, status.Errorf(codes.InvalidArgument, "invalid type %q", req.Type) + } + annotations := s.admin.NewDeploymentAnnotations(org, proj) + res, err = s.admin.Provision(ctx, &admin.ProvisionOptions{ + DeploymentID: depl.ID, + Type: typ, + Name: req.Name, + Provisioner: "", // Means it should find a suitable provisioner + Args: req.Args.AsMap(), + Annotations: annotations.ToMap(), + }) + if err != nil { + return nil, err + } + + return &adminv1.ProvisionResponse{ + Resource: provisionerResourceToPB(res), + }, nil +} + +func provisionerResourceToPB(i *database.ProvisionerResource) *adminv1.ProvisionerResource { + argsPB, err := structpb.NewStruct(i.Args) + if err != nil { + panic(err) + } + + return &adminv1.ProvisionerResource{ + Id: i.ID, + DeploymentId: i.DeploymentID, + Type: string(i.Type), + Name: i.Name, + Args: argsPB, + } +} diff --git a/docs/docs/manage/roles-permissions.md b/docs/docs/manage/roles-permissions.md index 21146f5c234..a289282368e 100644 --- a/docs/docs/manage/roles-permissions.md +++ b/docs/docs/manage/roles-permissions.md @@ -32,23 +32,25 @@ There are two roles available at the organization-level: **Viewer** and **Admin* There are two roles available at the project-level: **Viewer** and **Admin**. -| Permission | Description | Viewer | Admin | -| :------------------------- | :--------------------------------------------------------- | -----: | ----: | -| `read_project` | View basic info about the project | ✔ | ✔ | -| `manage_project` | Change project settings | | ✔ | -| `read_prod` | View dashboards deployed from the production (main) branch | ✔ | ✔ | -| `read_prod_status` | View logs for the production deployment | | ✔ | -| `manage_prod` | Trigger actions on the production deployment | | ✔ | -| `read_project_members` | View members of the project | | ✔ | -| `manage_project_members` | Add, remove or change roles of project members | | ✔ | -| `create_magic_auth_tokens` | Create shareable URLs | | ✔ | -| `manage_magic_auth_tokens` | Remove shareable URLs created by others | | ✔ | -| `create_reports` | Create and edit new scheduled reports | ✔ | ✔ | -| `manage_reports` | Edit and change scheduled reports created by others | | ✔ | -| `create_alerts` | Create and edit new alerts | ✔ | ✔ | -| `manage_alerts` | Edit and change alerts created by others | | ✔ | -| `create_bookmarks` | Create and edit new bookmarks | ✔ | ✔ | -| `manage_bookmarks` | Edit and change bookmarks created by others | | ✔ | +| Permission | Description | Viewer | Admin | +| :----------------------------- | :--------------------------------------------------------- | -----: | ----: | +| `read_project` | View basic info about the project | ✔ | ✔ | +| `manage_project` | Change project settings | | ✔ | +| `read_prod` | View dashboards deployed from the production (main) branch | ✔ | ✔ | +| `read_prod_status` | View logs for the production deployment | | ✔ | +| `manage_prod` | Trigger actions on the production deployment | | ✔ | +| `read_provisioner_resources` | View managed resources for the project | | ✔ | +| `manage_provisioner_resources` | Add or remove managed resources for the project | | ✔ | +| `read_project_members` | View members of the project | | ✔ | +| `manage_project_members` | Add, remove or change roles of project members | | ✔ | +| `create_magic_auth_tokens` | Create shareable URLs | | ✔ | +| `manage_magic_auth_tokens` | Remove shareable URLs created by others | | ✔ | +| `create_reports` | Create and edit new scheduled reports | ✔ | ✔ | +| `manage_reports` | Edit and change scheduled reports created by others | | ✔ | +| `create_alerts` | Create and edit new alerts | ✔ | ✔ | +| `manage_alerts` | Edit and change alerts created by others | | ✔ | +| `create_bookmarks` | Create and edit new bookmarks | ✔ | ✔ | +| `manage_bookmarks` | Edit and change bookmarks created by others | | ✔ |