From 4249445f89915ea4aaf74f9be4dd1417bef1c905 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 10:39:00 +0700 Subject: [PATCH 01/19] Change import path name of cosmos-sdk/types in cosmos/module.go --- cosmos/module.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cosmos/module.go b/cosmos/module.go index 46ab766a4..ca1daaf1a 100644 --- a/cosmos/module.go +++ b/cosmos/module.go @@ -5,7 +5,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/context" cosmoscodec "github.com/cosmos/cosmos-sdk/codec" - sdk "github.com/cosmos/cosmos-sdk/types" + cosmostypes "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/module" "github.com/gorilla/mux" "github.com/mesg-foundation/engine/codec" @@ -32,7 +32,7 @@ type AppModule struct { } // Querier is responsible to answer to ABCI queries. -type Querier func(request sdk.Request, path []string, req abci.RequestQuery) (res interface{}, err error) +type Querier func(request cosmostypes.Request, path []string, req abci.RequestQuery) (res interface{}, err error) // NewAppModuleBasic inits an AppModuleBasic using a name. func NewAppModuleBasic(name string) AppModuleBasic { @@ -91,7 +91,7 @@ func (AppModuleBasic) GetTxCmd(cdc *cosmoscodec.Codec) *cobra.Command { // ---------------------------------------------- // RegisterInvariants registers invariants to the registry. -func (m AppModule) RegisterInvariants(ir sdk.InvariantRegistry) {} +func (m AppModule) RegisterInvariants(ir cosmostypes.InvariantRegistry) {} // Route returns the route prefix for transaction of the module. func (m AppModule) Route() string { @@ -99,8 +99,8 @@ func (m AppModule) Route() string { } // NewHandler returns the handler used to apply transactions. -func (m AppModule) NewHandler() sdk.Handler { return m.handler +func (m AppModule) NewHandler() cosmostypes.Handler { } // QuerierRoute the route prefix for query of the module. @@ -109,37 +109,37 @@ func (m AppModule) QuerierRoute() string { } // NewQuerierHandler returns the handler used to reply ABCI query. -func (m AppModule) NewQuerierHandler() sdk.Querier { - return func(request sdk.Request, path []string, req abci.RequestQuery) ([]byte, sdk.Error) { +func (m AppModule) NewQuerierHandler() cosmostypes.Querier { + return func(request cosmostypes.Request, path []string, req abci.RequestQuery) ([]byte, cosmostypes.Error) { data, err := m.querier(request, path, req) if err != nil { - if errsdk, ok := err.(sdk.Error); ok { + if errsdk, ok := err.(cosmostypes.Error); ok { return nil, errsdk } - return nil, sdk.ErrInternal(err.Error()) + return nil, cosmostypes.ErrInternal(err.Error()) } res, err := codec.MarshalBinaryBare(data) if err != nil { - return nil, sdk.ErrInternal(err.Error()) + return nil, cosmostypes.ErrInternal(err.Error()) } return res, nil } } // BeginBlock is called at the beginning of the process of a new block. -func (m AppModule) BeginBlock(_ sdk.Request, _ abci.RequestBeginBlock) {} +func (m AppModule) BeginBlock(_ cosmostypes.Request, _ abci.RequestBeginBlock) {} // EndBlock is called at the end of the process of a new block. -func (m AppModule) EndBlock(sdk.Request, abci.RequestEndBlock) []abci.ValidatorUpdate { +func (m AppModule) EndBlock(cosmostypes.Request, abci.RequestEndBlock) []abci.ValidatorUpdate { return []abci.ValidatorUpdate{} } // InitGenesis initializes the genesis from a request and data. -func (m AppModule) InitGenesis(request sdk.Request, data json.RawMessage) []abci.ValidatorUpdate { +func (m AppModule) InitGenesis(request cosmostypes.Request, data json.RawMessage) []abci.ValidatorUpdate { return []abci.ValidatorUpdate{} } // ExportGenesis exports the current state of the app. -func (m AppModule) ExportGenesis(request sdk.Request) json.RawMessage { +func (m AppModule) ExportGenesis(request cosmostypes.Request) json.RawMessage { return []byte("{}") } From 04c9251484407bfd3569e8de26d45e258363e73f Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 10:45:34 +0700 Subject: [PATCH 02/19] Add cosmos handler wrapper that requires a hash to be returned. It emits the appropriate cosmos events and manage errors. --- cosmos/module.go | 29 ++++++++++++++++++++++++++--- cosmos/type.go | 5 +++++ sdk/instance/backend.go | 4 ++-- sdk/ownership/backend.go | 4 ++-- sdk/runner/backend.go | 15 +++++---------- sdk/service/backend.go | 10 ++++------ 6 files changed, 44 insertions(+), 23 deletions(-) create mode 100644 cosmos/type.go diff --git a/cosmos/module.go b/cosmos/module.go index ca1daaf1a..4dd660367 100644 --- a/cosmos/module.go +++ b/cosmos/module.go @@ -9,6 +9,7 @@ import ( "github.com/cosmos/cosmos-sdk/types/module" "github.com/gorilla/mux" "github.com/mesg-foundation/engine/codec" + "github.com/mesg-foundation/engine/hash" "github.com/spf13/cobra" abci "github.com/tendermint/tendermint/abci/types" ) @@ -27,10 +28,13 @@ type AppModuleBasic struct { // AppModule is a main element of an cosmos app. type AppModule struct { AppModuleBasic - handler sdk.Handler + handler Handler querier Querier } +// Handler defines the core of the state transition function of an application. +type Handler func(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) + // Querier is responsible to answer to ABCI queries. type Querier func(request cosmostypes.Request, path []string, req abci.RequestQuery) (res interface{}, err error) @@ -42,7 +46,7 @@ func NewAppModuleBasic(name string) AppModuleBasic { } // NewAppModule inits an AppModule using an AppModuleBasic, Handler and Querier. -func NewAppModule(moduleBasic AppModuleBasic, handler sdk.Handler, querier Querier) AppModule { +func NewAppModule(moduleBasic AppModuleBasic, handler Handler, querier Querier) AppModule { return AppModule{ AppModuleBasic: moduleBasic, handler: handler, @@ -99,8 +103,27 @@ func (m AppModule) Route() string { } // NewHandler returns the handler used to apply transactions. - return m.handler func (m AppModule) NewHandler() cosmostypes.Handler { + return func(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { + hash, err := m.handler(request, msg) + if err != nil { + if errsdk, ok := err.(cosmostypes.Error); ok { + return errsdk.Result() + } + return cosmostypes.ErrInternal(err.Error()).Result() + } + events := request.EventManager().Events() + if hash != nil { + events = events.AppendEvents([]cosmostypes.Event{ + cosmostypes.NewEvent(cosmostypes.EventTypeMessage, cosmostypes.NewAttribute(cosmostypes.AttributeKeyModule, m.name)), + cosmostypes.NewEvent(cosmostypes.EventTypeMessage, cosmostypes.NewAttribute(eventHashAttr, hash.String())), + }) + } + return cosmostypes.Result{ + Data: hash, + Events: events, + } + } } // QuerierRoute the route prefix for query of the module. diff --git a/cosmos/type.go b/cosmos/type.go new file mode 100644 index 000000000..75c8a5569 --- /dev/null +++ b/cosmos/type.go @@ -0,0 +1,5 @@ +package cosmos + +const ( + eventHashAttr = "hash" +) diff --git a/sdk/instance/backend.go b/sdk/instance/backend.go index 658cc76e0..69023fe11 100644 --- a/sdk/instance/backend.go +++ b/sdk/instance/backend.go @@ -32,9 +32,9 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { errmsg := fmt.Sprintf("Unrecognized instance Msg type: %v", msg.Type()) - return cosmostypes.ErrUnknownRequest(errmsg).Result() + return nil, cosmostypes.ErrUnknownRequest(errmsg) } func (s *Backend) querier(request cosmostypes.Request, path []string, req abci.RequestQuery) (interface{}, error) { diff --git a/sdk/ownership/backend.go b/sdk/ownership/backend.go index 586b222f8..a0fc01a25 100644 --- a/sdk/ownership/backend.go +++ b/sdk/ownership/backend.go @@ -31,9 +31,9 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { errmsg := fmt.Sprintf("Unrecognized ownership Msg type: %v", msg.Type()) - return cosmostypes.ErrUnknownRequest(errmsg).Result() + return nil, cosmostypes.ErrUnknownRequest(errmsg) } func (s *Backend) querier(request cosmostypes.Request, path []string, req abci.RequestQuery) (interface{}, error) { diff --git a/sdk/runner/backend.go b/sdk/runner/backend.go index 15dde027e..5baff0474 100644 --- a/sdk/runner/backend.go +++ b/sdk/runner/backend.go @@ -35,24 +35,19 @@ func NewBackend(appFactory *cosmos.AppFactory, instanceBack *instancesdk.Backend return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { switch msg := msg.(type) { case msgCreateRunner: run, err := s.Create(request, &msg) if err != nil { - return cosmostypes.ErrInternal(err.Error()).Result() - } - return cosmostypes.Result{ - Data: run.Hash, + return nil, err } + return run.Hash, nil case msgDeleteRunner: - if err := s.Delete(request, &msg); err != nil { - return cosmostypes.ErrInternal(err.Error()).Result() - } - return cosmostypes.Result{} + return nil, s.Delete(request, &msg) default: errmsg := fmt.Sprintf("Unrecognized runner Msg type: %v", msg.Type()) - return cosmostypes.ErrUnknownRequest(errmsg).Result() + return nil, cosmostypes.ErrUnknownRequest(errmsg) } } diff --git a/sdk/service/backend.go b/sdk/service/backend.go index 9fd0970ba..b2affff66 100644 --- a/sdk/service/backend.go +++ b/sdk/service/backend.go @@ -37,19 +37,17 @@ func NewBackend(appFactory *cosmos.AppFactory, ownerships *ownershipsdk.Backend) return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { switch msg := msg.(type) { case msgCreateService: srv, err := s.Create(request, &msg) if err != nil { - return cosmostypes.ErrInternal(err.Error()).Result() - } - return cosmostypes.Result{ - Data: srv.Hash, + return nil, err } + return srv.Hash, nil default: errmsg := fmt.Sprintf("Unrecognized service Msg type: %v", msg.Type()) - return cosmostypes.ErrUnknownRequest(errmsg).Result() + return nil, cosmostypes.ErrUnknownRequest(errmsg) } } From 444dfe81eff6e3586a24cefeaa9fda7daaeea1b3 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 10:49:34 +0700 Subject: [PATCH 03/19] add stream function to cosmos client --- cosmos/client.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ cosmos/type.go | 20 +++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/cosmos/client.go b/cosmos/client.go index 886b12181..8e0f65f9e 100644 --- a/cosmos/client.go +++ b/cosmos/client.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" + "math/rand" "time" "github.com/cosmos/cosmos-sdk/crypto/keys" sdktypes "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" "github.com/mesg-foundation/engine/codec" + "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/x/xreflect" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/node" @@ -129,3 +131,58 @@ func (c *Client) BuildAndBroadcastMsg(msg sdktypes.Msg, accName, accPassword str return nil, errors.New("i/o timeout") } } + +// Stream subscribes to the provided query and returns the hash of the matching ressources. +func (c *Client) Stream(query string) (<-chan hash.Hash, func() error, error) { + ctx, cancel := context.WithCancel(context.Background()) + // TODO: randStringRunes seems to complicated for this random string + subscriber := randStringRunes(8) // generates a new subscriber each time to be able to subscribe to the same query multiple time + eventStream, err := c.Subscribe(ctx, subscriber, query) + if err != nil { + return nil, func() error { + cancel() + return nil + }, err + } + closer := func() error { + err := c.Unsubscribe(ctx, subscriber, query) + if err != nil { + return err + } + cancel() + return nil + } + hashChan := make(chan hash.Hash) + go func() { + defer close(hashChan) + for event := range eventStream { + ressHashes := event.Events[eventHashKey()] + if len(ressHashes) != 1 { + // TODO: remove panic + panic(errors.New("there is no or more than one " + eventHashKey())) + } + hash, err := hash.Decode(ressHashes[0]) + if err != nil { + // TODO: remove panic + panic(err) + } + hashChan <- hash + } + }() + return hashChan, closer, nil +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func randStringRunes(n int) string { + b := make([]rune, n) + nbr := len(letterRunes) + for i := range b { + b[i] = letterRunes[rand.Intn(nbr)] + } + return string(b) +} diff --git a/cosmos/type.go b/cosmos/type.go index 75c8a5569..ec8b79f1c 100644 --- a/cosmos/type.go +++ b/cosmos/type.go @@ -1,5 +1,25 @@ package cosmos +import ( + "fmt" + "strings" + + cosmostypes "github.com/cosmos/cosmos-sdk/types" + sdktypes "github.com/cosmos/cosmos-sdk/types" +) + const ( eventHashAttr = "hash" ) + +func eventHashKey() string { + return strings.Join([]string{cosmostypes.EventTypeMessage, eventHashAttr}, ".") +} + +func EventActionQuery(msgType string) string { + return fmt.Sprintf("%s.%s='%s'", sdktypes.EventTypeMessage, sdktypes.AttributeKeyAction, msgType) +} + +func EventModuleQuery(module string) string { + return fmt.Sprintf("%s.%s='%s'", sdktypes.EventTypeMessage, sdktypes.AttributeKeyModule, module) +} From 054baf325f987c2ce07c81c39b30be1dbafa0951 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 17:21:27 +0700 Subject: [PATCH 04/19] Add executorHash to execution --- execution/execution.go | 3 +- execution/execution.pb.go | 80 ++++++++++++++++++---------------- execution/execution_test.go | 11 +++-- protobuf/types/execution.proto | 7 +++ 4 files changed, 57 insertions(+), 44 deletions(-) diff --git a/execution/execution.go b/execution/execution.go index 1c5ef187d..1de766942 100644 --- a/execution/execution.go +++ b/execution/execution.go @@ -6,7 +6,7 @@ import ( ) // New returns a new execution. It returns an error if inputs are invalid. -func New(processHash, instanceHash, parentHash, eventHash hash.Hash, stepID string, taskKey string, inputs *types.Struct, tags []string) *Execution { +func New(processHash, instanceHash, parentHash, eventHash hash.Hash, stepID string, taskKey string, inputs *types.Struct, tags []string, executorHash hash.Hash) *Execution { exec := &Execution{ ProcessHash: processHash, EventHash: eventHash, @@ -17,6 +17,7 @@ func New(processHash, instanceHash, parentHash, eventHash hash.Hash, stepID stri StepID: stepID, Tags: tags, Status: Status_Created, + ExecutorHash: executorHash, } exec.Hash = hash.Dump(exec) return exec diff --git a/execution/execution.pb.go b/execution/execution.pb.go index dbd890676..699e9c92c 100644 --- a/execution/execution.pb.go +++ b/execution/execution.pb.go @@ -93,10 +93,12 @@ type Execution struct { // processHash is the unique hash of the process associated to this execution. ProcessHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,11,opt,name=processHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"processHash" hash:"name:11"` // step of the process. - StepID string `protobuf:"bytes,12,opt,name=stepID,proto3" json:"stepID,omitempty" hash:"name:12"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + StepID string `protobuf:"bytes,12,opt,name=stepID,proto3" json:"stepID,omitempty" hash:"name:12"` + // runner that should execute this execution. + ExecutorHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,13,opt,name=executorHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"executorHash" hash:"name:13"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Execution) Reset() { *m = Execution{} } @@ -131,39 +133,40 @@ func init() { func init() { proto.RegisterFile("execution.proto", fileDescriptor_776e2c5022e94aef) } var fileDescriptor_776e2c5022e94aef = []byte{ - // 512 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x94, 0x51, 0x6a, 0xdb, 0x4c, - 0x10, 0xc7, 0xad, 0xd8, 0x91, 0xa3, 0xb1, 0xe3, 0xcf, 0xdf, 0x42, 0x41, 0xa4, 0x10, 0x8b, 0x7d, - 0x28, 0x22, 0x21, 0x52, 0xe3, 0x90, 0xb6, 0x04, 0x4a, 0x8b, 0x92, 0x94, 0x86, 0x52, 0x28, 0x32, - 0x7d, 0xe9, 0xdb, 0x5a, 0xde, 0x48, 0x22, 0xf1, 0xae, 0xd8, 0x5d, 0xb5, 0xcd, 0x8d, 0x7a, 0x94, - 0x9c, 0xa1, 0x50, 0x43, 0x7b, 0x84, 0x9c, 0xa0, 0xec, 0x2a, 0x4a, 0xac, 0x06, 0x4a, 0x09, 0x79, - 0xf3, 0xf8, 0x3f, 0xf3, 0x9b, 0xff, 0xce, 0x0c, 0x82, 0xff, 0xe8, 0x57, 0x9a, 0x94, 0x2a, 0xe7, - 0x2c, 0x28, 0x04, 0x57, 0x1c, 0xc1, 0x9c, 0xca, 0x34, 0x50, 0x17, 0x05, 0x95, 0x1b, 0x38, 0xe5, - 0x29, 0x0f, 0xcd, 0xff, 0xd3, 0xf2, 0x34, 0xd4, 0x91, 0x09, 0xcc, 0xaf, 0x2a, 0x7f, 0xe3, 0xf1, - 0x8d, 0x6c, 0x6a, 0x42, 0xa9, 0x44, 0x99, 0xa8, 0x4a, 0xc4, 0x3f, 0x6c, 0x70, 0x8e, 0xeb, 0x06, - 0x68, 0x02, 0x9d, 0x8c, 0xc8, 0xcc, 0xb5, 0x3c, 0xcb, 0xef, 0x47, 0xaf, 0x2e, 0x17, 0xa3, 0xd6, - 0xf7, 0xc5, 0x68, 0x3b, 0xcd, 0x55, 0x56, 0x4e, 0x83, 0x84, 0xcf, 0x43, 0xdd, 0x7b, 0xe7, 0x94, - 0x97, 0x6c, 0x46, 0x74, 0x45, 0x48, 0x59, 0x9a, 0x33, 0x1a, 0xea, 0xaa, 0xe0, 0x2d, 0x91, 0xd9, - 0xd5, 0x62, 0xb4, 0xa6, 0x83, 0x03, 0xbc, 0x83, 0x63, 0x03, 0x43, 0x33, 0x80, 0x82, 0x08, 0xca, - 0x94, 0xd6, 0xdd, 0x15, 0x83, 0x3e, 0xba, 0x1f, 0x7a, 0xbd, 0x42, 0x33, 0x32, 0xa7, 0x07, 0x63, - 0x1c, 0x2f, 0x71, 0xd1, 0x14, 0x1c, 0xfa, 0xb9, 0x6e, 0xd2, 0x7e, 0xa8, 0x26, 0x7b, 0x38, 0xbe, - 0xc5, 0xa2, 0x17, 0x60, 0x4b, 0x45, 0x54, 0x29, 0xdd, 0x8e, 0x67, 0xf9, 0x83, 0x31, 0x0a, 0x6e, - 0x57, 0x11, 0x4c, 0x8c, 0x12, 0xf5, 0x1b, 0x13, 0xb8, 0xce, 0x47, 0x19, 0xf4, 0x73, 0x26, 0x15, - 0x61, 0x09, 0x35, 0x06, 0x57, 0x1f, 0xca, 0xe0, 0x3e, 0x8e, 0x1b, 0x64, 0xb4, 0x0d, 0x5d, 0x45, - 0xe4, 0xd9, 0x3b, 0x7a, 0xe1, 0xda, 0x9e, 0xe5, 0x3b, 0xd1, 0xff, 0x7f, 0x54, 0x3c, 0xc3, 0x71, - 0x9d, 0x81, 0x5e, 0x83, 0x9d, 0xb3, 0xa2, 0x54, 0xd2, 0xed, 0x7a, 0x96, 0xdf, 0x1b, 0x3f, 0xaa, - 0x1e, 0x54, 0x1f, 0x4c, 0x30, 0x31, 0xa7, 0x72, 0x07, 0xf1, 0x1c, 0xc7, 0xd7, 0x75, 0xe8, 0x25, - 0x74, 0x79, 0xa9, 0x0c, 0x62, 0xed, 0x6f, 0x88, 0xe6, 0x58, 0xea, 0x1a, 0x84, 0x61, 0x95, 0x0a, - 0xc1, 0x85, 0xeb, 0x18, 0xaf, 0xcd, 0xac, 0x4a, 0x42, 0x4f, 0xa0, 0xa3, 0x48, 0x2a, 0x5d, 0xf0, - 0xda, 0xbe, 0x13, 0xa1, 0xab, 0xc5, 0x68, 0xb0, 0xe4, 0x65, 0xf7, 0x29, 0x8e, 0x8d, 0x8e, 0x52, - 0xe8, 0x15, 0x82, 0x27, 0x54, 0x4a, 0x33, 0xe2, 0x9e, 0x19, 0xf1, 0xf1, 0xfd, 0x46, 0xdc, 0xe8, - 0xb0, 0x8b, 0xe3, 0x65, 0x32, 0xda, 0xd2, 0x67, 0x40, 0x8b, 0x93, 0x23, 0xb7, 0x6f, 0x5c, 0xdf, - 0xb1, 0x34, 0x36, 0x8b, 0xd7, 0x19, 0x5b, 0xef, 0xc1, 0xae, 0x0e, 0x03, 0xf5, 0xa0, 0xfb, 0x91, - 0x9d, 0x31, 0xfe, 0x85, 0x0d, 0x5b, 0x3a, 0x38, 0x14, 0x94, 0x28, 0x3a, 0x1b, 0x5a, 0x68, 0x00, - 0x70, 0xc2, 0x3e, 0x08, 0x9e, 0x0a, 0x2a, 0xe5, 0x70, 0x05, 0xad, 0x83, 0x73, 0xc8, 0xe7, 0xc5, - 0x39, 0xd5, 0x72, 0x1b, 0x01, 0xd8, 0x6f, 0x48, 0x7e, 0x4e, 0x67, 0xc3, 0x4e, 0xb4, 0x7f, 0xf9, - 0x73, 0xb3, 0xf5, 0xed, 0xd7, 0xa6, 0xf5, 0xe9, 0x1f, 0x1e, 0x74, 0xf3, 0xe1, 0x98, 0xda, 0x66, - 0x1d, 0x7b, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x1f, 0xed, 0x3a, 0x78, 0x4c, 0x04, 0x00, 0x00, + // 526 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x94, 0xdf, 0x6a, 0x13, 0x41, + 0x14, 0xc6, 0xb3, 0x4d, 0xba, 0xe9, 0x9e, 0xfc, 0x31, 0x1e, 0x10, 0x96, 0x0a, 0x4d, 0x98, 0x0b, + 0x09, 0x2d, 0xdd, 0xb5, 0x09, 0x55, 0x29, 0x88, 0x92, 0xb6, 0x62, 0x11, 0x41, 0x36, 0x78, 0xe3, + 0xdd, 0x24, 0x99, 0x6e, 0x96, 0x36, 0x33, 0xcb, 0xcc, 0xac, 0xda, 0x37, 0xf2, 0x51, 0xf2, 0x0c, + 0x5e, 0x04, 0xf4, 0x11, 0xfa, 0x04, 0x32, 0xb3, 0x4d, 0x9b, 0xb5, 0x20, 0x52, 0x72, 0x97, 0x93, + 0xef, 0x9c, 0xdf, 0xf9, 0xf6, 0x9c, 0xb3, 0x0b, 0x8f, 0xd8, 0x77, 0x36, 0xce, 0x74, 0x22, 0x78, + 0x90, 0x4a, 0xa1, 0x05, 0xc2, 0x8c, 0xa9, 0x38, 0xd0, 0x57, 0x29, 0x53, 0xdb, 0x24, 0x16, 0xb1, + 0x08, 0xed, 0xff, 0xa3, 0xec, 0x3c, 0x34, 0x91, 0x0d, 0xec, 0xaf, 0x3c, 0x7f, 0xfb, 0xe9, 0xad, + 0x6c, 0x6b, 0x42, 0xa5, 0x65, 0x36, 0xd6, 0xb9, 0x48, 0xe6, 0x55, 0xf0, 0x4e, 0x97, 0x0d, 0x70, + 0x08, 0x95, 0x29, 0x55, 0x53, 0xdf, 0xe9, 0x38, 0xdd, 0xfa, 0xe0, 0xcd, 0x7c, 0xd1, 0x2e, 0xfd, + 0x5c, 0xb4, 0xf7, 0xe2, 0x44, 0x4f, 0xb3, 0x51, 0x30, 0x16, 0xb3, 0xd0, 0xf4, 0xde, 0x3f, 0x17, + 0x19, 0x9f, 0x50, 0x53, 0x11, 0x32, 0x1e, 0x27, 0x9c, 0x85, 0xa6, 0x2a, 0x78, 0x4f, 0xd5, 0xf4, + 0x7a, 0xd1, 0xde, 0x32, 0xc1, 0x11, 0xd9, 0x27, 0x91, 0x85, 0xe1, 0x04, 0x20, 0xa5, 0x92, 0x71, + 0x6d, 0x74, 0x7f, 0xc3, 0xa2, 0x4f, 0x1e, 0x86, 0x6e, 0xe4, 0x68, 0x4e, 0x67, 0xec, 0xa8, 0x47, + 0xa2, 0x15, 0x2e, 0x8e, 0xc0, 0x63, 0x5f, 0x97, 0x4d, 0xca, 0xeb, 0x6a, 0xd2, 0x27, 0xd1, 0x1d, + 0x16, 0x5f, 0x81, 0xab, 0x34, 0xd5, 0x99, 0xf2, 0x2b, 0x1d, 0xa7, 0xdb, 0xec, 0x61, 0x70, 0xb7, + 0x8a, 0x60, 0x68, 0x95, 0x41, 0xbd, 0x30, 0x81, 0x9b, 0x7c, 0x9c, 0x42, 0x3d, 0xe1, 0x4a, 0x53, + 0x3e, 0x66, 0xd6, 0xe0, 0xe6, 0xba, 0x0c, 0x1e, 0x92, 0xa8, 0x40, 0xc6, 0x3d, 0xa8, 0x6a, 0xaa, + 0x2e, 0x3e, 0xb0, 0x2b, 0xdf, 0xed, 0x38, 0x5d, 0x6f, 0xf0, 0xf8, 0xaf, 0x8a, 0x17, 0x24, 0x5a, + 0x66, 0xe0, 0x5b, 0x70, 0x13, 0x9e, 0x66, 0x5a, 0xf9, 0xd5, 0x8e, 0xd3, 0xad, 0xf5, 0x9e, 0xe4, + 0x0f, 0xb4, 0x3c, 0x98, 0x60, 0x68, 0x4f, 0xe5, 0x1e, 0xe2, 0x25, 0x89, 0x6e, 0xea, 0xf0, 0x35, + 0x54, 0x45, 0xa6, 0x2d, 0x62, 0xeb, 0x5f, 0x88, 0xe2, 0x58, 0x96, 0x35, 0x48, 0x60, 0x93, 0x49, + 0x29, 0xa4, 0xef, 0x59, 0xaf, 0xc5, 0xac, 0x5c, 0xc2, 0x67, 0x50, 0xd1, 0x34, 0x56, 0x3e, 0x74, + 0xca, 0x5d, 0x6f, 0x80, 0xd7, 0x8b, 0x76, 0x73, 0xc5, 0xcb, 0xc1, 0x73, 0x12, 0x59, 0x1d, 0x63, + 0xa8, 0xa5, 0x52, 0x8c, 0x99, 0x52, 0x76, 0xc4, 0x35, 0x3b, 0xe2, 0xd3, 0x87, 0x8d, 0xb8, 0xd0, + 0xe1, 0x80, 0x44, 0xab, 0x64, 0xdc, 0x35, 0x67, 0xc0, 0xd2, 0xb3, 0x13, 0xbf, 0x6e, 0x5d, 0xdf, + 0xb3, 0xd4, 0xb3, 0x8b, 0x37, 0x19, 0x98, 0x40, 0x3d, 0x7f, 0x7f, 0x85, 0xb4, 0xae, 0x1a, 0x6b, + 0x73, 0xd5, 0x27, 0x51, 0x01, 0xbd, 0xfb, 0x11, 0xdc, 0xfc, 0x06, 0xb1, 0x06, 0xd5, 0xcf, 0xfc, + 0x82, 0x8b, 0x6f, 0xbc, 0x55, 0x32, 0xc1, 0xb1, 0x64, 0x54, 0xb3, 0x49, 0xcb, 0xc1, 0x26, 0xc0, + 0x19, 0xff, 0x24, 0x45, 0x2c, 0x99, 0x52, 0xad, 0x0d, 0x6c, 0x80, 0x77, 0x2c, 0x66, 0xe9, 0x25, + 0x33, 0x72, 0x19, 0x01, 0xdc, 0x77, 0x34, 0xb9, 0x64, 0x93, 0x56, 0x65, 0x70, 0x38, 0xff, 0xb5, + 0x53, 0xfa, 0xf1, 0x7b, 0xc7, 0xf9, 0xf2, 0x1f, 0x2e, 0x6f, 0xbf, 0x51, 0x23, 0xd7, 0x6e, 0xbe, + 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, 0xea, 0xdb, 0xca, 0x27, 0xb7, 0x04, 0x00, 0x00, } func (this *Execution) Equal(that interface{}) bool { @@ -226,6 +229,9 @@ func (this *Execution) Equal(that interface{}) bool { if this.StepID != that1.StepID { return false } + if !this.ExecutorHash.Equal(that1.ExecutorHash) { + return false + } if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { return false } diff --git a/execution/execution_test.go b/execution/execution_test.go index 972f770be..20e6e9d6b 100644 --- a/execution/execution_test.go +++ b/execution/execution_test.go @@ -19,7 +19,7 @@ func TestNewFromService(t *testing.T) { tags = []string{"tag"} ) - execution := New(nil, hash, parentHash, eventHash, "", taskKey, nil, tags) + execution := New(nil, hash, parentHash, eventHash, "", taskKey, nil, tags, nil) require.NotNil(t, execution) require.Equal(t, hash, execution.InstanceHash) require.Equal(t, parentHash, execution.ParentHash) @@ -31,7 +31,7 @@ func TestNewFromService(t *testing.T) { } func TestExecute(t *testing.T) { - e := New(nil, nil, nil, nil, "", "", nil, nil) + e := New(nil, nil, nil, nil, "", "", nil, nil, nil) require.NoError(t, e.Execute()) require.Equal(t, Status_InProgress, e.Status) require.Error(t, e.Execute()) @@ -39,8 +39,7 @@ func TestExecute(t *testing.T) { func TestComplete(t *testing.T) { var output types.Struct - e := New(nil, nil, nil, nil, "", "", nil, nil) - + e := New(nil, nil, nil, nil, "", "", nil, nil, nil) e.Execute() require.NoError(t, e.Complete(&output)) require.Equal(t, Status_Completed, e.Status) @@ -50,7 +49,7 @@ func TestComplete(t *testing.T) { func TestFailed(t *testing.T) { err := errors.New("test") - e := New(nil, nil, nil, nil, "", "", nil, nil) + e := New(nil, nil, nil, nil, "", "", nil, nil, nil) e.Execute() require.NoError(t, e.Failed(err)) require.Equal(t, Status_Failed, e.Status) @@ -62,7 +61,7 @@ func TestExecutionHash(t *testing.T) { ids := make(map[string]bool) f := func(instanceHash, parentHash, eventID []byte, taskKey string, tags []string) bool { - e := New(nil, instanceHash, parentHash, eventID, "", taskKey, nil, tags) + e := New(nil, instanceHash, parentHash, eventID, "", taskKey, nil, tags, nil) if ids[string(e.Hash)] { return false } diff --git a/protobuf/types/execution.proto b/protobuf/types/execution.proto index 77301525b..32f737d1f 100644 --- a/protobuf/types/execution.proto +++ b/protobuf/types/execution.proto @@ -102,4 +102,11 @@ message Execution { string stepID = 12 [ (gogoproto.moretags) = 'hash:"name:12"' ]; + + // runner that should execute this execution. + bytes executorHash = 13 [ + (gogoproto.moretags) = 'hash:"name:13"', + (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", + (gogoproto.nullable) = false + ]; } From 7fb58e45eb80ee9c05396ac153b57bb28a8fe11c Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 17:22:37 +0700 Subject: [PATCH 05/19] Move execution sdk to cosmos sdk --- sdk/execution/backend.go | 200 +++++++++++++++++++++ sdk/execution/codec.go | 8 + sdk/execution/execution.go | 193 --------------------- sdk/execution/execution_listener.go | 95 ---------- sdk/execution/execution_listener_test.go | 98 ----------- sdk/execution/execution_test.go | 212 ++++++++++++----------- sdk/execution/msgs.go | 100 +++++++++++ sdk/execution/sdk.go | 195 +++++++++++++++++++++ 8 files changed, 611 insertions(+), 490 deletions(-) create mode 100644 sdk/execution/backend.go create mode 100644 sdk/execution/codec.go delete mode 100644 sdk/execution/execution.go delete mode 100644 sdk/execution/execution_listener.go delete mode 100644 sdk/execution/execution_listener_test.go create mode 100644 sdk/execution/msgs.go create mode 100644 sdk/execution/sdk.go diff --git a/sdk/execution/backend.go b/sdk/execution/backend.go new file mode 100644 index 000000000..9ad681180 --- /dev/null +++ b/sdk/execution/backend.go @@ -0,0 +1,200 @@ +package executionsdk + +import ( + "errors" + "fmt" + + cosmostypes "github.com/cosmos/cosmos-sdk/types" + "github.com/mesg-foundation/engine/codec" + "github.com/mesg-foundation/engine/cosmos" + "github.com/mesg-foundation/engine/execution" + "github.com/mesg-foundation/engine/hash" + "github.com/mesg-foundation/engine/protobuf/api" + "github.com/mesg-foundation/engine/protobuf/types" + instancesdk "github.com/mesg-foundation/engine/sdk/instance" + processsdk "github.com/mesg-foundation/engine/sdk/process" + servicesdk "github.com/mesg-foundation/engine/sdk/service" + abci "github.com/tendermint/tendermint/abci/types" +) + +const backendName = "execution" + +// Backend is the execution backend. +type Backend struct { + storeKey *cosmostypes.KVStoreKey + serviceBack *servicesdk.Backend + instanceBack *instancesdk.Backend + processSDK *processsdk.Process // TODO: to replace by process backend when process sdk is running on cosmos. +} + +// NewBackend returns the backend of the execution sdk. +func NewBackend(appFactory *cosmos.AppFactory, serviceBack *servicesdk.Backend, instanceBack *instancesdk.Backend) *Backend { + backend := &Backend{ + storeKey: cosmostypes.NewKVStoreKey(backendName), + serviceBack: serviceBack, + instanceBack: instanceBack, + } + appBackendBasic := cosmos.NewAppModuleBasic(backendName) + appBackend := cosmos.NewAppModule(appBackendBasic, backend.handler, backend.querier) + appFactory.RegisterModule(appBackend) + appFactory.RegisterStoreKey(backend.storeKey) + + return backend +} + +// SetProcessSDK sets the process sdk. +// TODO: this is a hack and will be remove when process sdk is running on cosmos +func (s *Backend) SetProcessSDK(processSDK *processsdk.Process) { + s.processSDK = processSDK +} + +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { + switch msg := msg.(type) { + case msgCreateExecution: + exec, err := s.Create(request, msg) + if err != nil { + return nil, err + } + return exec.Hash, nil + case msgUpdateExecution: + exec, err := s.Update(request, msg) + if err != nil { + return nil, err + } + return exec.Hash, nil + default: + errmsg := fmt.Sprintf("Unrecognized execution Msg type: %v", msg.Type()) + return nil, cosmostypes.ErrUnknownRequest(errmsg) + } +} + +func (s *Backend) querier(request cosmostypes.Request, path []string, req abci.RequestQuery) (interface{}, error) { + switch path[0] { + case "get": + hash, err := hash.Decode(path[1]) + if err != nil { + return nil, err + } + return s.Get(request, hash) + case "list": + return s.List(request) + default: + return nil, errors.New("unknown execution query endpoint" + path[0]) + } +} + +// Create creates a new execution from definition. +func (s *Backend) Create(request cosmostypes.Request, msg msgCreateExecution) (*execution.Execution, error) { + inst, err := s.instanceBack.Get(request, msg.Request.InstanceHash) + if err != nil { + return nil, err + } + srv, err := s.serviceBack.Get(request, inst.ServiceHash) + if err != nil { + return nil, err + } + if !msg.Request.ProcessHash.IsZero() { + if _, err := s.processSDK.Get(msg.Request.ProcessHash); err != nil { + return nil, err + } + } + if err := srv.RequireTaskInputs(msg.Request.TaskKey, msg.Request.Inputs); err != nil { + return nil, err + } + exec := execution.New( + msg.Request.ProcessHash, + msg.Request.InstanceHash, + msg.Request.ParentHash, + msg.Request.EventHash, + msg.Request.StepID, + msg.Request.TaskKey, + msg.Request.Inputs, + msg.Request.Tags, + msg.Request.ExecutorHash, + ) + if err := exec.Execute(); err != nil { + return nil, err + } + store := request.KVStore(s.storeKey) + value, err := codec.MarshalBinaryBare(exec) + if err != nil { + return nil, err + } + store.Set(exec.Hash, value) + return exec, nil +} + +// Update updates a new execution from definition. +func (s *Backend) Update(request cosmostypes.Request, msg msgUpdateExecution) (*execution.Execution, error) { + store := request.KVStore(s.storeKey) + if !store.Has(msg.Request.Hash) { + return nil, fmt.Errorf("execution %q doesn't exist", msg.Request.Hash) + } + var exec *execution.Execution + if err := codec.UnmarshalBinaryBare(store.Get(msg.Request.Hash), &exec); err != nil { + return nil, err + } + switch res := msg.Request.Result.(type) { + case *api.UpdateExecutionRequest_Outputs: + if err := s.validateExecutionOutput(request, exec.InstanceHash, exec.TaskKey, res.Outputs); err != nil { + if err1 := exec.Failed(err); err1 != nil { + return nil, err1 + } + } else if err := exec.Complete(res.Outputs); err != nil { + return nil, err + } + case *api.UpdateExecutionRequest_Error: + if err := exec.Failed(errors.New(res.Error)); err != nil { + return nil, err + } + default: + return nil, errors.New("no execution result supplied") + } + value, err := codec.MarshalBinaryBare(exec) + if err != nil { + return nil, err + } + store.Set(exec.Hash, value) + return exec, nil +} + +func (s *Backend) validateExecutionOutput(request cosmostypes.Request, instanceHash hash.Hash, taskKey string, outputs *types.Struct) error { + inst, err := s.instanceBack.Get(request, instanceHash) + if err != nil { + return err + } + srv, err := s.serviceBack.Get(request, inst.ServiceHash) + if err != nil { + return err + } + return srv.RequireTaskOutputs(taskKey, outputs) +} + +// Get returns the execution that matches given hash. +func (s *Backend) Get(request cosmostypes.Request, hash hash.Hash) (*execution.Execution, error) { + var exec *execution.Execution + store := request.KVStore(s.storeKey) + if !store.Has(hash) { + return nil, fmt.Errorf("execution %q not found", hash) + } + return exec, codec.UnmarshalBinaryBare(store.Get(hash), &exec) +} + +// List returns all executions. +func (s *Backend) List(request cosmostypes.Request) ([]*execution.Execution, error) { + var ( + execs []*execution.Execution + iter = request.KVStore(s.storeKey).Iterator(nil, nil) + ) + for iter.Valid() { + var exec *execution.Execution + value := iter.Value() + if err := codec.UnmarshalBinaryBare(value, &exec); err != nil { + return nil, err + } + execs = append(execs, exec) + iter.Next() + } + iter.Close() + return execs, nil +} diff --git a/sdk/execution/codec.go b/sdk/execution/codec.go new file mode 100644 index 000000000..6039f7a40 --- /dev/null +++ b/sdk/execution/codec.go @@ -0,0 +1,8 @@ +package executionsdk + +import "github.com/mesg-foundation/engine/codec" + +func init() { + codec.RegisterConcrete(msgCreateExecution{}, "execution/create", nil) + codec.RegisterConcrete(msgUpdateExecution{}, "execution/update", nil) +} diff --git a/sdk/execution/execution.go b/sdk/execution/execution.go deleted file mode 100644 index 06d9920e0..000000000 --- a/sdk/execution/execution.go +++ /dev/null @@ -1,193 +0,0 @@ -package executionsdk - -import ( - "fmt" - - "github.com/cskr/pubsub" - "github.com/mesg-foundation/engine/database" - "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/hash" - "github.com/mesg-foundation/engine/protobuf/types" - instancesdk "github.com/mesg-foundation/engine/sdk/instance" - processesdk "github.com/mesg-foundation/engine/sdk/process" - servicesdk "github.com/mesg-foundation/engine/sdk/service" -) - -const ( - // streamTopic is topic used to broadcast executions. - streamTopic = "execution-stream" - topic = "Execution" -) - -// Execution exposes execution APIs of MESG. -type Execution struct { - ps *pubsub.PubSub - service *servicesdk.SDK - instance *instancesdk.SDK - process *processesdk.Process - execDB database.ExecutionDB -} - -// New creates a new Execution SDK with given options. -func New(ps *pubsub.PubSub, service *servicesdk.SDK, instance *instancesdk.SDK, process *processesdk.Process, execDB database.ExecutionDB) *Execution { - return &Execution{ - ps: ps, - service: service, - instance: instance, - process: process, - execDB: execDB, - } -} - -// Get returns execution that matches given hash. -func (e *Execution) Get(hash hash.Hash) (*execution.Execution, error) { - return e.execDB.Find(hash) -} - -// GetStream returns execution that matches given hash. -func (e *Execution) GetStream(f *Filter) *Listener { - l := NewListener(e.ps, streamTopic, f) - go l.Listen() - return l -} - -// Update updates execution that matches given hash. -func (e *Execution) Update(executionHash hash.Hash, outputs *types.Struct, reterr error) error { - exec, err := e.processExecution(executionHash, outputs, reterr) - if err != nil { - return err - } - - go e.ps.Pub(exec, streamTopic) - go e.ps.Pub(exec, subTopic(exec.InstanceHash)) - return nil -} - -// processExecution processes execution and marks it as complated or failed. -func (e *Execution) processExecution(executionHash hash.Hash, outputs *types.Struct, reterr error) (*execution.Execution, error) { - tx, err := e.execDB.OpenTransaction() - if err != nil { - return nil, err - } - - exec, err := tx.Find(executionHash) - if err != nil { - tx.Discard() - return nil, err - } - - if reterr != nil { - if err := exec.Failed(reterr); err != nil { - tx.Discard() - return nil, err - } - } else { - err := e.validateExecutionOutput(exec.InstanceHash, exec.TaskKey, outputs) - if err != nil { - if err1 := exec.Failed(err); err1 != nil { - tx.Discard() - return nil, err1 - } - } else if err := exec.Complete(outputs); err != nil { - tx.Discard() - return nil, err - } - } - - if err := tx.Save(exec); err != nil { - tx.Discard() - return nil, err - } - - if err := tx.Commit(); err != nil { - tx.Discard() - return nil, err - } - - return exec, nil -} - -func (e *Execution) validateExecutionOutput(instanceHash hash.Hash, taskKey string, outputs *types.Struct) error { - i, err := e.instance.Get(instanceHash) - if err != nil { - return err - } - - s, err := e.service.Get(i.ServiceHash) - if err != nil { - return err - } - - return s.RequireTaskOutputs(taskKey, outputs) -} - -// Execute executes a task tasKey with inputData and tags for service serviceID. -func (e *Execution) Execute(processHash, instanceHash, eventHash, parentHash hash.Hash, stepID string, taskKey string, inputData *types.Struct, tags []string) (executionHash hash.Hash, err error) { - if parentHash != nil && eventHash != nil { - return nil, fmt.Errorf("cannot have both parent and event hash") - } - if parentHash == nil && eventHash == nil { - return nil, fmt.Errorf("should have at least an event hash or parent hash") - } - // a task should be executed only if task's service is running. - instance, err := e.instance.Get(instanceHash) - if err != nil { - return nil, err - } - - s, err := e.service.Get(instance.ServiceHash) - if err != nil { - return nil, err - } - - if !processHash.IsZero() { - _, err := e.process.Get(processHash) - if err != nil { - return nil, err - } - } - - if err := s.RequireTaskInputs(taskKey, inputData); err != nil { - return nil, err - } - - exec := execution.New(processHash, instance.Hash, parentHash, eventHash, stepID, taskKey, inputData, tags) - if err := exec.Execute(); err != nil { - return nil, err - } - if err = e.execDB.Save(exec); err != nil { - return nil, err - } - - go e.ps.Pub(exec, streamTopic) - go e.ps.Pub(exec, subTopic(instance.Hash)) - return exec.Hash, nil -} - -// Listen listens executions on instance. -func (e *Execution) Listen(instanceHash hash.Hash, f *Filter) (*Listener, error) { - inst, err := e.instance.Get(instanceHash) - if err != nil { - return nil, err - } - - service, err := e.service.Get(inst.ServiceHash) - if err != nil { - return nil, err - } - - if f != nil && f.HasTaskKey() { - if _, err := service.GetTask(f.TaskKey); err != nil { - return nil, err - } - } - - l := NewListener(e.ps, subTopic(inst.Hash), f) - go l.Listen() - return l, nil -} - -// subTopic returns the topic to listen for tasks from this service. -func subTopic(instanceHash hash.Hash) string { - return instanceHash.String() + "." + topic -} diff --git a/sdk/execution/execution_listener.go b/sdk/execution/execution_listener.go deleted file mode 100644 index c5a337fc2..000000000 --- a/sdk/execution/execution_listener.go +++ /dev/null @@ -1,95 +0,0 @@ -package executionsdk - -import ( - "github.com/cskr/pubsub" - "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/hash" - "github.com/mesg-foundation/engine/x/xstrings" -) - -// Filter store fileds for matching executions. -type Filter struct { - Statuses []execution.Status - InstanceHash hash.Hash - TaskKey string - Tags []string -} - -// Match matches execution. -func (f *Filter) Match(e *execution.Execution) bool { - if f == nil { - return true - } - - if !f.InstanceHash.IsZero() && !f.InstanceHash.Equal(e.InstanceHash) { - return false - } - - if f.TaskKey != "" && f.TaskKey != "*" && f.TaskKey != e.TaskKey { - return false - } - - match := len(f.Statuses) == 0 - for _, status := range f.Statuses { - if status == e.Status { - match = true - break - } - } - - if !match { - return false - } - - for _, tag := range f.Tags { - if !xstrings.SliceContains(e.Tags, tag) { - return false - } - } - return true -} - -// HasTaskKey returns true if task key is set to specified value. -func (f *Filter) HasTaskKey() bool { - return f != nil && f.TaskKey != "" && f.TaskKey != "*" -} - -// Listener provides functionalities to listen MESG tasks. -type Listener struct { - // Channel receives matching executions for tasks. - C chan *execution.Execution - - ps *pubsub.PubSub - topic string - c chan interface{} - - filter *Filter -} - -// NewListener creates a new Listener with given sdk. -func NewListener(ps *pubsub.PubSub, topic string, f *Filter) *Listener { - return &Listener{ - C: make(chan *execution.Execution, 1), - ps: ps, - topic: topic, - c: ps.Sub(topic), - filter: f, - } -} - -// Close stops listening for events. -func (l *Listener) Close() { - go func() { - l.ps.Unsub(l.c, l.topic) - close(l.C) - }() -} - -// Listen listens executions that match filter. -func (l *Listener) Listen() { - for v := range l.c { - if e := v.(*execution.Execution); l.filter.Match(e) { - l.C <- e - } - } -} diff --git a/sdk/execution/execution_listener_test.go b/sdk/execution/execution_listener_test.go deleted file mode 100644 index f14b717da..000000000 --- a/sdk/execution/execution_listener_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package executionsdk - -import ( - "testing" - - "github.com/cskr/pubsub" - "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/hash" - "github.com/stretchr/testify/assert" -) - -func TestFilter(t *testing.T) { - var tests = []struct { - f *Filter - e *execution.Execution - match bool - }{ - { - nil, - nil, - true, - }, - { - &Filter{}, - &execution.Execution{}, - true, - }, - { - &Filter{InstanceHash: hash.Int(1)}, - &execution.Execution{InstanceHash: hash.Int(1)}, - true, - }, - { - &Filter{InstanceHash: hash.Int(1)}, - &execution.Execution{InstanceHash: hash.Int(2)}, - false, - }, - { - &Filter{Statuses: []execution.Status{execution.Status_Created}}, - &execution.Execution{Status: execution.Status_Created}, - true, - }, - { - &Filter{Statuses: []execution.Status{execution.Status_Created}}, - &execution.Execution{Status: execution.Status_InProgress}, - false, - }, - { - &Filter{TaskKey: "0"}, - &execution.Execution{TaskKey: "0"}, - true, - }, - { - &Filter{TaskKey: "*"}, - &execution.Execution{TaskKey: "0"}, - true, - }, - { - &Filter{TaskKey: "0"}, - &execution.Execution{TaskKey: "1"}, - false, - }, - { - &Filter{Tags: []string{"0"}}, - &execution.Execution{Tags: []string{"0"}}, - true, - }, - { - &Filter{Tags: []string{"0", "1"}}, - &execution.Execution{Tags: []string{"0"}}, - false, - }, - } - - for i, tt := range tests { - assert.Equal(t, tt.match, tt.f.Match(tt.e), i) - } -} - -func TestListener(t *testing.T) { - topic := "test-topic" - testExecution := &execution.Execution{TaskKey: "0"} - ps := pubsub.New(0) - el := NewListener(ps, topic, &Filter{TaskKey: "0"}) - - go func() { - ps.Pub(&execution.Execution{TaskKey: "1"}, topic) - ps.Pub(testExecution, topic) - }() - go el.Listen() - - recvExecution := <-el.C - assert.Equal(t, testExecution, recvExecution) - - el.Close() - _, ok := <-el.C - assert.False(t, ok) -} diff --git a/sdk/execution/execution_test.go b/sdk/execution/execution_test.go index 9d8a0fb78..b96a3a1b9 100644 --- a/sdk/execution/execution_test.go +++ b/sdk/execution/execution_test.go @@ -3,112 +3,116 @@ package executionsdk import ( "testing" + "github.com/mesg-foundation/engine/execution" "github.com/mesg-foundation/engine/hash" - "github.com/stretchr/testify/require" + "github.com/mesg-foundation/engine/protobuf/api" + "gotest.tools/assert" ) -// TODO: restore test after refactor create of cosmos node for testing. - -// func newTesting(t *testing.T) (*Execution, *apiTesting) { -// serviceStore, err := store.NewLevelDBStore(servicedbname) -// require.NoError(t, err) -// db := database.NewServiceDB(serviceStore) - -// instanceStore, err := store.NewLevelDBStore(instdbname) -// require.NoError(t, err) -// instDB := database.NewInstanceDB(instanceStore) -// instance := instancesdk.New(nil) - -// execDB, err := database.NewExecutionDB(execdbname) -// require.NoError(t, err) - -// processDB, err := database.NewProcessDB(processdbname) -// require.NoError(t, err) -// process := processesdk.New(instance, processDB) - -// sdk := New(pubsub.New(0), nil, instance, process, execDB) - -// return sdk, &apiTesting{ -// T: t, -// serviceDB: db, -// executionDB: execDB, -// instanceDB: instDB, -// processDB: processDB, -// } -// } - -// var hs1 = hash.Int(1) - -// var testService = &service.Service{ -// Name: "1", -// Sid: "2", -// Hash: hs1, -// Tasks: []*service.Service_Task{ -// {Key: "4"}, -// }, -// Dependencies: []*service.Service_Dependency{ -// {Key: "5"}, -// }, -// } - -// func TestGet(t *testing.T) { -// sdk, at := newTesting(t) -// defer at.close() -// exec := execution.New(nil, nil, nil, nil, "", "", nil, nil) -// require.NoError(t, sdk.execDB.Save(exec)) -// got, err := sdk.Get(exec.Hash) -// require.NoError(t, err) -// require.True(t, exec.Equal(got)) -// } - -// func TestGetStream(t *testing.T) { -// sdk, at := newTesting(t) -// defer at.close() - -// exec := execution.New(nil, nil, nil, nil, "", "", nil, nil) -// exec.Status = execution.Status_InProgress - -// require.NoError(t, sdk.execDB.Save(exec)) - -// stream := sdk.GetStream(nil) -// defer stream.Close() - -// go sdk.ps.Pub(exec, streamTopic) -// exec.Status = execution.Status_Failed -// exec.Error = "exec-error" -// require.Equal(t, exec, <-stream.C) -// } - -// func TestExecute(t *testing.T) { -// var testInstance = &instance.Instance{ -// Hash: hash.Int(2), -// ServiceHash: hs1, -// } - -// sdk, at := newTesting(t) -// defer at.close() - -// require.NoError(t, at.serviceDB.Save(testService)) -// require.NoError(t, at.instanceDB.Save(testInstance)) - -// _, err := sdk.Execute(nil, testInstance.Hash, hash.Int(1), nil, "", testService.Tasks[0].Key, nil, nil) -// require.NoError(t, err) - -// // not existing instance -// _, err = sdk.Execute(nil, hash.Int(3), hash.Int(1), nil, "", testService.Tasks[0].Key, nil, nil) -// require.Error(t, err) -// } - -// func TestExecuteInvalidTaskKey(t *testing.T) { -// sdk, at := newTesting(t) -// defer at.close() - -// require.NoError(t, at.serviceDB.Save(testService)) - -// _, err := sdk.Execute(nil, hs1, hash.Int(1), nil, "", "-", nil, nil) -// require.Error(t, err) -// } +func TestFilter(t *testing.T) { + var tests = []struct { + f *api.StreamExecutionRequest_Filter + e *execution.Execution + match bool + }{ + { + nil, + nil, + true, + }, + { + &api.StreamExecutionRequest_Filter{}, + &execution.Execution{}, + true, + }, + { + &api.StreamExecutionRequest_Filter{InstanceHash: hash.Int(1)}, + &execution.Execution{InstanceHash: hash.Int(1)}, + true, + }, + { + &api.StreamExecutionRequest_Filter{InstanceHash: hash.Int(1)}, + &execution.Execution{InstanceHash: hash.Int(2)}, + false, + }, + { + &api.StreamExecutionRequest_Filter{ExecutorHash: hash.Int(1)}, + &execution.Execution{ExecutorHash: hash.Int(1)}, + true, + }, + { + &api.StreamExecutionRequest_Filter{ExecutorHash: hash.Int(1)}, + &execution.Execution{ExecutorHash: hash.Int(2)}, + false, + }, + { + &api.StreamExecutionRequest_Filter{Statuses: []execution.Status{execution.Status_Created}}, + &execution.Execution{Status: execution.Status_Created}, + true, + }, + { + &api.StreamExecutionRequest_Filter{Statuses: []execution.Status{execution.Status_Created}}, + &execution.Execution{Status: execution.Status_InProgress}, + false, + }, + { + &api.StreamExecutionRequest_Filter{TaskKey: "0"}, + &execution.Execution{TaskKey: "0"}, + true, + }, + { + &api.StreamExecutionRequest_Filter{TaskKey: "*"}, + &execution.Execution{TaskKey: "0"}, + true, + }, + { + &api.StreamExecutionRequest_Filter{TaskKey: "0"}, + &execution.Execution{TaskKey: "1"}, + false, + }, + { + &api.StreamExecutionRequest_Filter{Tags: []string{"0"}}, + &execution.Execution{Tags: []string{"0"}}, + true, + }, + { + &api.StreamExecutionRequest_Filter{Tags: []string{"0", "1"}}, + &execution.Execution{Tags: []string{"0"}}, + false, + }, + } + + for i, tt := range tests { + assert.Equal(t, tt.match, match(tt.f, tt.e), i) + } +} -func TestSubTopic(t *testing.T) { - require.Equal(t, subTopic(hash.Hash{0}), "1.Execution") +func TestValidateFilter(t *testing.T) { + var tests = []struct { + f *api.StreamExecutionRequest_Filter + isError bool + }{ + { + nil, + false, + }, + { + &api.StreamExecutionRequest_Filter{}, + false, + }, + { + &api.StreamExecutionRequest_Filter{ + TaskKey: "not-exist", + }, + false, + }, + } + s := SDK{} + for i, tt := range tests { + if tt.isError { + assert.Error(t, s.validateFilter(tt.f), "", i) + } else { + assert.NilError(t, s.validateFilter(tt.f), i) + } + } } diff --git a/sdk/execution/msgs.go b/sdk/execution/msgs.go new file mode 100644 index 000000000..7452cbc7d --- /dev/null +++ b/sdk/execution/msgs.go @@ -0,0 +1,100 @@ +package executionsdk + +import ( + cosmostypes "github.com/cosmos/cosmos-sdk/types" + "github.com/mesg-foundation/engine/codec" + "github.com/mesg-foundation/engine/protobuf/api" +) + +// msgCreateExecution defines a state transition to create a execution. +type msgCreateExecution struct { + Request *api.CreateExecutionRequest `json:"request"` + Signer cosmostypes.AccAddress `json:"signer"` +} + +const msgCreateExecutionType = "create_execution" +const msgUpdateExecutionType = "update_execution" + +// newMsgCreateExecution is a constructor function for msgCreateExecution. +func newMsgCreateExecution(req *api.CreateExecutionRequest, signer cosmostypes.AccAddress) *msgCreateExecution { + return &msgCreateExecution{ + Request: req, + Signer: signer, + } +} + +// Route should return the name of the module. +func (msg msgCreateExecution) Route() string { + return backendName +} + +// Type returns the action. +func (msg msgCreateExecution) Type() string { + return msgCreateExecutionType +} + +// ValidateBasic runs stateless checks on the message. +func (msg msgCreateExecution) ValidateBasic() cosmostypes.Error { + if !msg.Request.ParentHash.IsZero() && !msg.Request.EventHash.IsZero() { + return cosmostypes.ErrInternal("cannot have both parent and event hash") + } + if msg.Request.ParentHash.IsZero() && msg.Request.EventHash.IsZero() { + return cosmostypes.ErrInternal("should have at least an event hash or parent hash") + } + if msg.Request.ExecutorHash.IsZero() { + return cosmostypes.ErrInternal("should have a executor hash") + } + return nil +} + +// GetSignBytes encodes the message for signing. +func (msg msgCreateExecution) GetSignBytes() []byte { + return cosmostypes.MustSortJSON(codec.MustMarshalJSON(msg)) +} + +// GetSigners defines whose signature is required. +func (msg msgCreateExecution) GetSigners() []cosmostypes.AccAddress { + return []cosmostypes.AccAddress{msg.Signer} +} + +// msgUpdateExecution defines a state transition to update a execution. +type msgUpdateExecution struct { + Request *api.UpdateExecutionRequest `json:"request"` + Executor cosmostypes.AccAddress `json:"executor"` +} + +// newMsgUpdateExecution is a constructor function for msgUpdateExecution. +func newMsgUpdateExecution(req *api.UpdateExecutionRequest, executor cosmostypes.AccAddress) *msgUpdateExecution { + return &msgUpdateExecution{ + Request: req, + Executor: executor, + } +} + +// Route should return the name of the module. +func (msg msgUpdateExecution) Route() string { + return backendName +} + +// Type returns the action. +func (msg msgUpdateExecution) Type() string { + return msgUpdateExecutionType +} + +// ValidateBasic runs stateless checks on the message. +func (msg msgUpdateExecution) ValidateBasic() cosmostypes.Error { + if msg.Executor.Empty() { + return cosmostypes.ErrInvalidAddress("executor is missing") + } + return nil +} + +// GetSignBytes encodes the message for signing. +func (msg msgUpdateExecution) GetSignBytes() []byte { + return cosmostypes.MustSortJSON(codec.MustMarshalJSON(msg)) +} + +// GetSigners defines whose signature is required. +func (msg msgUpdateExecution) GetSigners() []cosmostypes.AccAddress { + return []cosmostypes.AccAddress{msg.Executor} +} diff --git a/sdk/execution/sdk.go b/sdk/execution/sdk.go new file mode 100644 index 000000000..d9bac8891 --- /dev/null +++ b/sdk/execution/sdk.go @@ -0,0 +1,195 @@ +package executionsdk + +import ( + "fmt" + + cosmostypes "github.com/cosmos/cosmos-sdk/types" + "github.com/mesg-foundation/engine/cosmos" + "github.com/mesg-foundation/engine/execution" + "github.com/mesg-foundation/engine/hash" + "github.com/mesg-foundation/engine/instance" + "github.com/mesg-foundation/engine/protobuf/api" + "github.com/mesg-foundation/engine/runner" + accountsdk "github.com/mesg-foundation/engine/sdk/account" + instancesdk "github.com/mesg-foundation/engine/sdk/instance" + runnersdk "github.com/mesg-foundation/engine/sdk/runner" + servicesdk "github.com/mesg-foundation/engine/sdk/service" + "github.com/mesg-foundation/engine/x/xstrings" + "github.com/tendermint/tendermint/mempool" +) + +// SDK is the execution sdk. +type SDK struct { + client *cosmos.Client + accountSDK *accountsdk.SDK + serviceSDK *servicesdk.SDK + instanceSDK *instancesdk.SDK + runnerSDK *runnersdk.SDK +} + +// New returns the execution sdk. +func New(client *cosmos.Client, accountSDK *accountsdk.SDK, serviceSDK *servicesdk.SDK, instanceSDK *instancesdk.SDK, runnerSDK *runnersdk.SDK) *SDK { + sdk := &SDK{ + client: client, + accountSDK: accountSDK, + serviceSDK: serviceSDK, + instanceSDK: instanceSDK, + runnerSDK: runnerSDK, + } + return sdk +} + +// Create creates a new execution. +func (s *SDK) Create(req *api.CreateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) { + account, err := s.accountSDK.Get(accountName) + if err != nil { + return nil, err + } + signer, err := cosmostypes.AccAddressFromBech32(account.Address) + if err != nil { + return nil, err + } + msg := newMsgCreateExecution(req, signer) + tx, err := s.client.BuildAndBroadcastMsg(msg, accountName, accountPassword) + if err != nil { + if err == mempool.ErrTxInCache { + return nil, fmt.Errorf("execution already exists: %w", err) + } + return nil, err + } + return s.Get(tx.Data) +} + +// Update updates a execution. +func (s *SDK) Update(req *api.UpdateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) { + account, err := s.accountSDK.Get(accountName) + if err != nil { + return nil, err + } + executor, err := cosmostypes.AccAddressFromBech32(account.Address) + if err != nil { + return nil, err + } + msg := newMsgUpdateExecution(req, executor) + tx, err := s.client.BuildAndBroadcastMsg(msg, accountName, accountPassword) + if err != nil { + if err == mempool.ErrTxInCache { + return nil, fmt.Errorf("execution already exists: %w", err) + } + return nil, err + } + return s.Get(tx.Data) +} + +// Get returns the execution that matches given hash. +func (s *SDK) Get(hash hash.Hash) (*execution.Execution, error) { + var execution execution.Execution + if err := s.client.Query("custom/"+backendName+"/get/"+hash.String(), nil, &execution); err != nil { + return nil, err + } + return &execution, nil +} + +// List returns all executions. +func (s *SDK) List() ([]*execution.Execution, error) { + var executions []*execution.Execution + if err := s.client.Query("custom/"+backendName+"/list", nil, &executions); err != nil { + return nil, err + } + return executions, nil +} + +// Stream returns execution that matches given hash. +func (s *SDK) Stream(req *api.StreamExecutionRequest) (<-chan *execution.Execution, func() error, error) { + if err := s.validateFilter(req.Filter); err != nil { + return nil, func() error { return nil }, err + } + stream, closer, err := s.client.Stream(cosmos.EventModuleQuery(backendName)) + if err != nil { + return nil, closer, err + } + execChan := make(chan *execution.Execution) + go func() { + defer close(execChan) + for hash := range stream { + exec, err := s.Get(hash) + if err != nil { + // TODO: remove panic + panic(err) + } + if match(req.Filter, exec) { + execChan <- exec + } + } + }() + return execChan, closer, nil +} + +func (s *SDK) validateFilter(f *api.StreamExecutionRequest_Filter) error { + if f == nil { + return nil + } + var err error + var run *runner.Runner + if !f.ExecutorHash.IsZero() { + if run, err = s.runnerSDK.Get(f.ExecutorHash); err != nil { + return err + } + } + var inst *instance.Instance + if !f.InstanceHash.IsZero() { + if inst, err = s.instanceSDK.Get(f.InstanceHash); err != nil { + return err + } + } + if (f.TaskKey == "" || f.TaskKey == "*") || (inst == nil && run == nil) { + return nil + } + // check task key if at least instance or runner is set + if inst == nil && run != nil { + inst, err = s.instanceSDK.Get(run.InstanceHash) + if err != nil { + return err + } + } + srv, err := s.serviceSDK.Get(inst.ServiceHash) + if err != nil { + return err + } + if _, err := srv.GetTask(f.TaskKey); err != nil { + return err + } + return nil +} + +// Match matches an execution against a filter. +func match(f *api.StreamExecutionRequest_Filter, e *execution.Execution) bool { + if f == nil { + return true + } + if !f.ExecutorHash.IsZero() && !f.ExecutorHash.Equal(e.ExecutorHash) { + return false + } + if !f.InstanceHash.IsZero() && !f.InstanceHash.Equal(e.InstanceHash) { + return false + } + if f.TaskKey != "" && f.TaskKey != "*" && f.TaskKey != e.TaskKey { + return false + } + match := len(f.Statuses) == 0 + for _, status := range f.Statuses { + if status == e.Status { + match = true + break + } + } + if !match { + return false + } + for _, tag := range f.Tags { + if !xstrings.SliceContains(e.Tags, tag) { + return false + } + } + return true +} From 262b87037e5216666f806ed8a43e23e07def7628 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 17:23:56 +0700 Subject: [PATCH 06/19] Update codebase to reflect modification of previous commit --- core/main.go | 23 ++-- orchestrator/mocks/ExecutionSDK.go | 54 +++++--- orchestrator/mocks/RunnerSDK.go | 36 +++++ orchestrator/orchestrator.go | 60 ++++++--- orchestrator/orchestrator_test.go | 16 ++- orchestrator/type.go | 21 ++- protobuf/api/codec.go | 11 ++ protobuf/api/execution.pb.go | 209 +++++++++++++++++++++++------ protobuf/api/execution.proto | 35 +++++ scripts/build-mocks.sh | 1 + sdk/backend.go | 4 + sdk/sdk.go | 6 +- server/grpc/api/execution.go | 59 ++++---- server/grpc/api/execution_test.go | 48 ------- 14 files changed, 399 insertions(+), 184 deletions(-) create mode 100644 orchestrator/mocks/RunnerSDK.go create mode 100644 protobuf/api/codec.go delete mode 100644 server/grpc/api/execution_test.go diff --git a/core/main.go b/core/main.go index bce0ceee3..c8596ba66 100644 --- a/core/main.go +++ b/core/main.go @@ -27,18 +27,9 @@ import ( db "github.com/tendermint/tm-db" ) -func initDatabases(cfg *config.Config) (*database.LevelDBExecutionDB, *database.LevelDBProcessDB, error) { - // init execution db. - executionDB, err := database.NewExecutionDB(filepath.Join(cfg.Path, cfg.Database.ExecutionRelativePath)) - if err != nil { - return nil, nil, err - } +func initDatabases(cfg *config.Config) (*database.LevelDBProcessDB, error) { // init process db. - processDB, err := database.NewProcessDB(filepath.Join(cfg.Path, cfg.Database.ProcessRelativePath)) - if err != nil { - return nil, nil, err - } - return executionDB, processDB, nil + return database.NewProcessDB(filepath.Join(cfg.Path, cfg.Database.ProcessRelativePath)) } func stopRunningServices(sdk *enginesdk.SDK, cfg *config.Config, address string) error { @@ -132,7 +123,7 @@ func main() { logger.Init(cfg.Log.Format, cfg.Log.Level, cfg.Log.ForceColors) // init databases - executionDB, processDB, err := initDatabases(cfg) + processDB, err := initDatabases(cfg) if err != nil { logrus.WithField("module", "main").Fatalln(err) } @@ -155,7 +146,7 @@ func main() { // register the backend modules to the app factory. // TODO: this is a mandatory call so it should return a new types required by cosmos.NewApp - enginesdk.NewBackend(appFactory) + backend := enginesdk.NewBackend(appFactory) // init cosmos app app, err := cosmos.NewApp(appFactory) @@ -191,7 +182,9 @@ func main() { client := cosmos.NewClient(node, kb, genesis.ChainID) // init sdk - sdk := enginesdk.New(client, kb, executionDB, processDB, container, cfg.Name, strconv.Itoa(port), cfg.IpfsEndpoint) + sdk := enginesdk.New(client, kb, processDB, container, cfg.Name, strconv.Itoa(port), cfg.IpfsEndpoint) + // TODO: this is a hack and will be remove when process sdk is running on cosmos + backend.Execution.SetProcessSDK(sdk.Process) // start tendermint node logrus.WithField("module", "main").WithField("seeds", cfg.Tendermint.Config.P2P.Seeds).Info("starting tendermint node") @@ -211,7 +204,7 @@ func main() { }() logrus.WithField("module", "main").Info("starting process engine") - s := orchestrator.New(sdk.Event, sdk.Execution, sdk.Process) + s := orchestrator.New(sdk.Event, sdk.Execution, sdk.Process, sdk.Runner, cfg.Account.Name, cfg.Account.Password) go func() { if err := s.Start(); err != nil { logrus.WithField("module", "main").Fatalln(err) diff --git a/orchestrator/mocks/ExecutionSDK.go b/orchestrator/mocks/ExecutionSDK.go index de37e64d9..d7fd49bf8 100644 --- a/orchestrator/mocks/ExecutionSDK.go +++ b/orchestrator/mocks/ExecutionSDK.go @@ -2,34 +2,32 @@ package mocks +import api "github.com/mesg-foundation/engine/protobuf/api" import execution "github.com/mesg-foundation/engine/execution" -import executionsdk "github.com/mesg-foundation/engine/sdk/execution" import hash "github.com/mesg-foundation/engine/hash" import mock "github.com/stretchr/testify/mock" -import types "github.com/mesg-foundation/engine/protobuf/types" - // ExecutionSDK is an autogenerated mock type for the ExecutionSDK type type ExecutionSDK struct { mock.Mock } -// Execute provides a mock function with given fields: processHash, instanceHash, eventHash, parentHash, stepID, taskKey, inputData, tags -func (_m *ExecutionSDK) Execute(processHash hash.Hash, instanceHash hash.Hash, eventHash hash.Hash, parentHash hash.Hash, stepID string, taskKey string, inputData *types.Struct, tags []string) (hash.Hash, error) { - ret := _m.Called(processHash, instanceHash, eventHash, parentHash, stepID, taskKey, inputData, tags) +// Create provides a mock function with given fields: req, accountName, accountPassword +func (_m *ExecutionSDK) Create(req *api.CreateExecutionRequest, accountName string, accountPassword string) (*execution.Execution, error) { + ret := _m.Called(req, accountName, accountPassword) - var r0 hash.Hash - if rf, ok := ret.Get(0).(func(hash.Hash, hash.Hash, hash.Hash, hash.Hash, string, string, *types.Struct, []string) hash.Hash); ok { - r0 = rf(processHash, instanceHash, eventHash, parentHash, stepID, taskKey, inputData, tags) + var r0 *execution.Execution + if rf, ok := ret.Get(0).(func(*api.CreateExecutionRequest, string, string) *execution.Execution); ok { + r0 = rf(req, accountName, accountPassword) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(hash.Hash) + r0 = ret.Get(0).(*execution.Execution) } } var r1 error - if rf, ok := ret.Get(1).(func(hash.Hash, hash.Hash, hash.Hash, hash.Hash, string, string, *types.Struct, []string) error); ok { - r1 = rf(processHash, instanceHash, eventHash, parentHash, stepID, taskKey, inputData, tags) + if rf, ok := ret.Get(1).(func(*api.CreateExecutionRequest, string, string) error); ok { + r1 = rf(req, accountName, accountPassword) } else { r1 = ret.Error(1) } @@ -60,18 +58,34 @@ func (_m *ExecutionSDK) Get(_a0 hash.Hash) (*execution.Execution, error) { return r0, r1 } -// GetStream provides a mock function with given fields: f -func (_m *ExecutionSDK) GetStream(f *executionsdk.Filter) *executionsdk.Listener { - ret := _m.Called(f) +// Stream provides a mock function with given fields: req +func (_m *ExecutionSDK) Stream(req *api.StreamExecutionRequest) (<-chan *execution.Execution, func() error, error) { + ret := _m.Called(req) - var r0 *executionsdk.Listener - if rf, ok := ret.Get(0).(func(*executionsdk.Filter) *executionsdk.Listener); ok { - r0 = rf(f) + var r0 <-chan *execution.Execution + if rf, ok := ret.Get(0).(func(*api.StreamExecutionRequest) <-chan *execution.Execution); ok { + r0 = rf(req) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*executionsdk.Listener) + r0 = ret.Get(0).(<-chan *execution.Execution) + } + } + + var r1 func() error + if rf, ok := ret.Get(1).(func(*api.StreamExecutionRequest) func() error); ok { + r1 = rf(req) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(func() error) } } - return r0 + var r2 error + if rf, ok := ret.Get(2).(func(*api.StreamExecutionRequest) error); ok { + r2 = rf(req) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } diff --git a/orchestrator/mocks/RunnerSDK.go b/orchestrator/mocks/RunnerSDK.go new file mode 100644 index 000000000..53d2031f1 --- /dev/null +++ b/orchestrator/mocks/RunnerSDK.go @@ -0,0 +1,36 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +import runner "github.com/mesg-foundation/engine/runner" +import runnersdk "github.com/mesg-foundation/engine/sdk/runner" + +// RunnerSDK is an autogenerated mock type for the RunnerSDK type +type RunnerSDK struct { + mock.Mock +} + +// List provides a mock function with given fields: f +func (_m *RunnerSDK) List(f *runnersdk.Filter) ([]*runner.Runner, error) { + ret := _m.Called(f) + + var r0 []*runner.Runner + if rf, ok := ret.Get(0).(func(*runnersdk.Filter) []*runner.Runner); ok { + r0 = rf(f) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*runner.Runner) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*runnersdk.Filter) error); ok { + r1 = rf(f) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index d75778932..ebff280ad 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -2,23 +2,28 @@ package orchestrator import ( "fmt" + "math/rand" "github.com/mesg-foundation/engine/event" "github.com/mesg-foundation/engine/execution" "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/process" + "github.com/mesg-foundation/engine/protobuf/api" "github.com/mesg-foundation/engine/protobuf/types" - executionsdk "github.com/mesg-foundation/engine/sdk/execution" + runnersdk "github.com/mesg-foundation/engine/sdk/runner" "github.com/sirupsen/logrus" ) // New creates a new Process instance -func New(event EventSDK, execution ExecutionSDK, process ProcessSDK) *Orchestrator { +func New(event EventSDK, execution ExecutionSDK, process ProcessSDK, runner RunnerSDK, accountName, accountPassword string) *Orchestrator { return &Orchestrator{ - event: event, - execution: execution, - process: process, - ErrC: make(chan error), + event: event, + execution: execution, + process: process, + runner: runner, + ErrC: make(chan error), + accountName: accountName, + accountPassword: accountPassword, } } @@ -28,18 +33,28 @@ func (s *Orchestrator) Start() error { return fmt.Errorf("process orchestrator already running") } s.eventStream = s.event.GetStream(nil) - s.executionStream = s.execution.GetStream(&executionsdk.Filter{ - Statuses: []execution.Status{execution.Status_Completed}, + executionStream, closer, err := s.execution.Stream(&api.StreamExecutionRequest{ + Filter: &api.StreamExecutionRequest_Filter{ + Statuses: []execution.Status{execution.Status_Completed}, + }, }) + if err != nil { + if err := closer(); err != nil { + panic(err) + } + return err + } + s.executionStream = executionStream for { select { case event := <-s.eventStream.C: go s.execute(s.eventFilter(event), nil, event, event.Data) - case execution := <-s.executionStream.C: + case execution := <-s.executionStream: go s.execute(s.resultFilter(execution), execution, nil, execution.Outputs) go s.execute(s.dependencyFilter(execution), execution, nil, execution.Outputs) } } + // TODO: manage closer } func (s *Orchestrator) eventFilter(event *event.Event) func(wf *process.Process, node *process.Process_Node) (bool, error) { @@ -105,7 +120,6 @@ func (s *Orchestrator) executeNode(wf *process.Process, n *process.Process_Node, logrus.WithField("module", "orchestrator"). WithField("nodeID", n.ID()). WithField("type", fmt.Sprintf("%T", n)).Debug("process process") - if task := n.GetTask(); task != nil { // This returns directly because a task cannot process its children. // Children will be processed only when the execution is done and the dependencies are resolved @@ -146,7 +160,6 @@ func (s *Orchestrator) processMap(mapping *process.Process_Node_Map, wf *process if err != nil { return nil, err } - if node.GetTask() != nil { value, err := s.resolveInput(wf.Hash, exec, ref.NodeKey, ref.Key) if err != nil { @@ -174,20 +187,37 @@ func (s *Orchestrator) resolveInput(wfHash hash.Hash, exec *execution.Execution, } return s.resolveInput(wfHash, parent, nodeKey, outputKey) } - return exec.Outputs.Fields[outputKey], nil } func (s *Orchestrator) processTask(task *process.Process_Node_Task, wf *process.Process, exec *execution.Execution, event *event.Event, data *types.Struct) error { var eventHash, execHash hash.Hash - if event != nil { eventHash = event.Hash } if exec != nil { execHash = exec.Hash } - - _, err := s.execution.Execute(wf.Hash, task.InstanceHash, eventHash, execHash, task.Key, task.TaskKey, data, nil) + executors, err := s.runner.List(&runnersdk.Filter{ + InstanceHash: task.InstanceHash, + }) + if err != nil { + return err + } + if len(executors) == 0 { + return fmt.Errorf("no runner is running instance %q", task.InstanceHash) + } + executor := executors[rand.Intn(len(executors))] + _, err = s.execution.Create(&api.CreateExecutionRequest{ + ProcessHash: wf.Hash, + InstanceHash: task.InstanceHash, + EventHash: eventHash, + ParentHash: execHash, + StepID: task.Key, + TaskKey: task.TaskKey, + Inputs: data, + ExecutorHash: executor.Hash, + Tags: nil, + }, s.accountName, s.accountPassword) return err } diff --git a/orchestrator/orchestrator_test.go b/orchestrator/orchestrator_test.go index ea2fd3969..ccc41a981 100644 --- a/orchestrator/orchestrator_test.go +++ b/orchestrator/orchestrator_test.go @@ -10,12 +10,13 @@ import ( "github.com/mesg-foundation/engine/orchestrator/mocks" "github.com/mesg-foundation/engine/process" "github.com/mesg-foundation/engine/protobuf/types" + "github.com/mesg-foundation/engine/runner" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestFilter(t *testing.T) { - o := New(&mocks.EventSDK{}, &mocks.ExecutionSDK{}, &mocks.ProcessSDK{}) + o := New(&mocks.EventSDK{}, &mocks.ExecutionSDK{}, &mocks.ProcessSDK{}, &mocks.RunnerSDK{}, "", "") p := process.Process{ Hash: hash.Int(1), Nodes: []*process.Process_Node{ @@ -144,7 +145,7 @@ func TestFilter(t *testing.T) { } func TestFindNode(t *testing.T) { - o := New(&mocks.EventSDK{}, &mocks.ExecutionSDK{}, &mocks.ProcessSDK{}) + o := New(&mocks.EventSDK{}, &mocks.ExecutionSDK{}, &mocks.ProcessSDK{}, &mocks.RunnerSDK{}, "", "") data := &process.Process{ Hash: hash.Int(1), Nodes: []*process.Process_Node{ @@ -190,7 +191,7 @@ func TestFindNode(t *testing.T) { func TestResolveInput(t *testing.T) { e := &mocks.ExecutionSDK{} - o := New(&mocks.EventSDK{}, e, &mocks.ProcessSDK{}) + o := New(&mocks.EventSDK{}, e, &mocks.ProcessSDK{}, &mocks.RunnerSDK{}, "", "") exec := &execution.Execution{ ProcessHash: hash.Int(2), StepID: "2", @@ -234,8 +235,10 @@ func TestResolveInput(t *testing.T) { func TestProcessTask(t *testing.T) { e := &mocks.ExecutionSDK{} - e.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once().Return(nil, nil) - o := New(&mocks.EventSDK{}, e, &mocks.ProcessSDK{}) + e.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once().Return(nil, nil) + r := &mocks.RunnerSDK{} + r.On("List", mock.Anything).Once().Return([]*runner.Runner{{Hash: hash.Int(1)}}, nil) + o := New(&mocks.EventSDK{}, e, &mocks.ProcessSDK{}, r, "", "") err := o.processTask(&process.Process_Node_Task{ InstanceHash: hash.Int(1), Key: "-", @@ -248,7 +251,8 @@ func TestProcessTask(t *testing.T) { Fields: map[string]*types.Value{}, }) require.NoError(t, err) - e.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once().Return(nil, fmt.Errorf("error")) + e.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Once().Return(nil, fmt.Errorf("error")) + r.On("List", mock.Anything).Once().Return([]*runner.Runner{{Hash: hash.Int(1)}}, nil) err = o.processTask(&process.Process_Node_Task{ InstanceHash: hash.Int(1), Key: "-", diff --git a/orchestrator/type.go b/orchestrator/type.go index 94c6bc788..7ead469c8 100644 --- a/orchestrator/type.go +++ b/orchestrator/type.go @@ -4,16 +4,17 @@ import ( "github.com/mesg-foundation/engine/execution" "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/process" - "github.com/mesg-foundation/engine/protobuf/types" + "github.com/mesg-foundation/engine/protobuf/api" + "github.com/mesg-foundation/engine/runner" eventsdk "github.com/mesg-foundation/engine/sdk/event" - executionsdk "github.com/mesg-foundation/engine/sdk/execution" + runnersdk "github.com/mesg-foundation/engine/sdk/runner" ) // ExecutionSDK execution interface needed for the orchestrator type ExecutionSDK interface { - GetStream(f *executionsdk.Filter) *executionsdk.Listener + Stream(req *api.StreamExecutionRequest) (<-chan *execution.Execution, func() error, error) Get(hash hash.Hash) (*execution.Execution, error) - Execute(processHash, instanceHash, eventHash, parentHash hash.Hash, stepID string, taskKey string, inputData *types.Struct, tags []string) (executionHash hash.Hash, err error) + Create(req *api.CreateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) } // EventSDK event interface needed for the orchestrator @@ -26,15 +27,25 @@ type ProcessSDK interface { List() ([]*process.Process, error) } +// RunnerSDK is the interface of the runner sdk needed for the orchestrator +type RunnerSDK interface { + List(f *runnersdk.Filter) ([]*runner.Runner, error) +} + // Orchestrator manages the executions based on the definition of the processes type Orchestrator struct { event EventSDK eventStream *eventsdk.Listener execution ExecutionSDK - executionStream *executionsdk.Listener + executionStream <-chan *execution.Execution process ProcessSDK + runner RunnerSDK + ErrC chan error + + accountName string + accountPassword string } diff --git a/protobuf/api/codec.go b/protobuf/api/codec.go new file mode 100644 index 000000000..eaf536293 --- /dev/null +++ b/protobuf/api/codec.go @@ -0,0 +1,11 @@ +package api + +import ( + "github.com/mesg-foundation/engine/codec" +) + +func init() { + codec.RegisterInterface((*isUpdateExecutionRequest_Result)(nil), nil) + codec.RegisterConcrete(&UpdateExecutionRequest_Outputs{}, "mesg.api.UpdateExecutionRequest_Outputs", nil) + codec.RegisterConcrete(&UpdateExecutionRequest_Error{}, "mesg.api.UpdateExecutionRequest_Error", nil) +} diff --git a/protobuf/api/execution.pb.go b/protobuf/api/execution.pb.go index ed5b80cda..47b196efc 100644 --- a/protobuf/api/execution.pb.go +++ b/protobuf/api/execution.pb.go @@ -34,6 +34,11 @@ type CreateExecutionRequest struct { TaskKey string `protobuf:"bytes,2,opt,name=taskKey,proto3" json:"taskKey,omitempty"` Inputs *types.Struct `protobuf:"bytes,3,opt,name=inputs,proto3" json:"inputs,omitempty"` Tags []string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty"` + ParentHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,5,opt,name=parentHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"parentHash"` + EventHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,6,opt,name=eventHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"eventHash"` + ProcessHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,7,opt,name=processHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"processHash"` + StepID string `protobuf:"bytes,8,opt,name=stepID,proto3" json:"stepID,omitempty"` + ExecutorHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,9,opt,name=executorHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"executorHash"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -84,6 +89,13 @@ func (m *CreateExecutionRequest) GetTags() []string { return nil } +func (m *CreateExecutionRequest) GetStepID() string { + if m != nil { + return m.StepID + } + return "" +} + // CreateExecutionResponse defines response for execution creation. type CreateExecutionResponse struct { // Execution's hash. @@ -199,10 +211,12 @@ type StreamExecutionRequest_Filter struct { // taskKey to filter executions. TaskKey string `protobuf:"bytes,3,opt,name=taskKey,proto3" json:"taskKey,omitempty"` // tags to filter executions. All tags needs to be present in the execution. - Tags []string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Tags []string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty"` + // Executor's hash to filter executions. + ExecutorHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,5,opt,name=executorHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"executorHash"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *StreamExecutionRequest_Filter) Reset() { *m = StreamExecutionRequest_Filter{} } @@ -363,6 +377,77 @@ func (m *UpdateExecutionResponse) XXX_DiscardUnknown() { var xxx_messageInfo_UpdateExecutionResponse proto.InternalMessageInfo +// The request's data for the `List` API. +type ListExecutionRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListExecutionRequest) Reset() { *m = ListExecutionRequest{} } +func (m *ListExecutionRequest) String() string { return proto.CompactTextString(m) } +func (*ListExecutionRequest) ProtoMessage() {} +func (*ListExecutionRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_96e2c86581f82f05, []int{6} +} +func (m *ListExecutionRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListExecutionRequest.Unmarshal(m, b) +} +func (m *ListExecutionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListExecutionRequest.Marshal(b, m, deterministic) +} +func (m *ListExecutionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListExecutionRequest.Merge(m, src) +} +func (m *ListExecutionRequest) XXX_Size() int { + return xxx_messageInfo_ListExecutionRequest.Size(m) +} +func (m *ListExecutionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ListExecutionRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ListExecutionRequest proto.InternalMessageInfo + +// The response's data for the `List` API. +type ListExecutionResponse struct { + // List of executions that match the request's filters. + Executions []*execution.Execution `protobuf:"bytes,1,rep,name=executions,proto3" json:"executions,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListExecutionResponse) Reset() { *m = ListExecutionResponse{} } +func (m *ListExecutionResponse) String() string { return proto.CompactTextString(m) } +func (*ListExecutionResponse) ProtoMessage() {} +func (*ListExecutionResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_96e2c86581f82f05, []int{7} +} +func (m *ListExecutionResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListExecutionResponse.Unmarshal(m, b) +} +func (m *ListExecutionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListExecutionResponse.Marshal(b, m, deterministic) +} +func (m *ListExecutionResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListExecutionResponse.Merge(m, src) +} +func (m *ListExecutionResponse) XXX_Size() int { + return xxx_messageInfo_ListExecutionResponse.Size(m) +} +func (m *ListExecutionResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ListExecutionResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ListExecutionResponse proto.InternalMessageInfo + +func (m *ListExecutionResponse) GetExecutions() []*execution.Execution { + if m != nil { + return m.Executions + } + return nil +} + func init() { proto.RegisterType((*CreateExecutionRequest)(nil), "mesg.api.CreateExecutionRequest") proto.RegisterType((*CreateExecutionResponse)(nil), "mesg.api.CreateExecutionResponse") @@ -371,45 +456,55 @@ func init() { proto.RegisterType((*StreamExecutionRequest_Filter)(nil), "mesg.api.StreamExecutionRequest.Filter") proto.RegisterType((*UpdateExecutionRequest)(nil), "mesg.api.UpdateExecutionRequest") proto.RegisterType((*UpdateExecutionResponse)(nil), "mesg.api.UpdateExecutionResponse") + proto.RegisterType((*ListExecutionRequest)(nil), "mesg.api.ListExecutionRequest") + proto.RegisterType((*ListExecutionResponse)(nil), "mesg.api.ListExecutionResponse") } func init() { proto.RegisterFile("protobuf/api/execution.proto", fileDescriptor_96e2c86581f82f05) } var fileDescriptor_96e2c86581f82f05 = []byte{ - // 519 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x93, 0xcf, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0xed, 0x38, 0x75, 0x93, 0x69, 0xc5, 0x61, 0x51, 0x53, 0x13, 0xfe, 0xd4, 0xf8, 0x42, - 0x24, 0xd4, 0x35, 0xa4, 0x67, 0x84, 0x14, 0x54, 0x12, 0x89, 0x03, 0x92, 0x23, 0x84, 0xc4, 0x01, - 0x69, 0x93, 0x4e, 0x1c, 0x8b, 0xc6, 0x6b, 0xbc, 0xb3, 0x12, 0x7d, 0x16, 0x1e, 0x03, 0x89, 0x2b, - 0x57, 0x4e, 0x3c, 0x00, 0x87, 0x3e, 0x0b, 0xca, 0x6e, 0x92, 0xa6, 0x8d, 0xdb, 0x0b, 0xed, 0xcd, - 0xb3, 0xf3, 0xed, 0xb7, 0xfb, 0xed, 0xfc, 0x0c, 0x8f, 0x8a, 0x52, 0x92, 0x1c, 0xe9, 0x49, 0x2c, - 0x8a, 0x2c, 0xc6, 0x6f, 0x38, 0xd6, 0x94, 0xc9, 0x9c, 0x9b, 0x65, 0xd6, 0x98, 0xa1, 0x4a, 0xb9, - 0x28, 0xb2, 0x76, 0x94, 0xca, 0x54, 0xc6, 0x2b, 0xf1, 0xbc, 0x32, 0x85, 0xf9, 0xb2, 0xea, 0xf6, - 0xc3, 0x55, 0x9b, 0xce, 0x0a, 0x54, 0xb1, 0xa2, 0x52, 0x8f, 0x69, 0xd1, 0x7c, 0x72, 0xa5, 0x79, - 0xe5, 0xa8, 0xe8, 0x8f, 0x0b, 0xad, 0x37, 0x25, 0x0a, 0xc2, 0xe3, 0x65, 0x27, 0xc1, 0xaf, 0x1a, - 0x15, 0xb1, 0x8f, 0xb0, 0x9b, 0xe5, 0x8a, 0x44, 0x3e, 0xc6, 0x81, 0x50, 0xd3, 0xc0, 0x0d, 0xdd, - 0xce, 0x6e, 0xef, 0xe8, 0xf7, 0xf9, 0x81, 0xf3, 0xf7, 0xfc, 0xe0, 0x79, 0x9a, 0xd1, 0x54, 0x8f, - 0xf8, 0x58, 0xce, 0xe2, 0xf9, 0x75, 0x0f, 0x27, 0x52, 0xe7, 0x27, 0x62, 0x6e, 0x10, 0x63, 0x9e, - 0x66, 0x39, 0xc6, 0x53, 0xa1, 0xa6, 0x7c, 0xbe, 0x35, 0xb9, 0x64, 0xc4, 0x02, 0xd8, 0x26, 0xa1, - 0xbe, 0xbc, 0xc3, 0xb3, 0xa0, 0x16, 0xba, 0x9d, 0x66, 0xb2, 0x2c, 0xd9, 0x21, 0xf8, 0x59, 0x5e, - 0x68, 0x52, 0x81, 0x17, 0xba, 0x9d, 0x9d, 0xee, 0x1e, 0x37, 0x2f, 0xb1, 0xcc, 0xc0, 0x87, 0x26, - 0x5a, 0xb2, 0x10, 0x31, 0x06, 0x75, 0x12, 0xa9, 0x0a, 0xea, 0xa1, 0xd7, 0x69, 0x26, 0xe6, 0x3b, - 0x1a, 0xc1, 0xfe, 0x46, 0x1e, 0x55, 0xc8, 0x5c, 0x21, 0xeb, 0x43, 0x7d, 0xfa, 0x9f, 0x41, 0x8c, - 0x41, 0xf4, 0x19, 0xee, 0xf7, 0x91, 0x36, 0x1e, 0xec, 0xd6, 0xfc, 0xbf, 0xd7, 0xa0, 0x35, 0xa4, - 0x12, 0xc5, 0x6c, 0xe3, 0x8c, 0xd7, 0xe0, 0x4f, 0xb2, 0x53, 0xc2, 0xd2, 0x9c, 0xb2, 0xd3, 0x7d, - 0xc6, 0x97, 0xac, 0xf0, 0xea, 0x1d, 0xfc, 0xad, 0x91, 0x27, 0x8b, 0x6d, 0xed, 0x5f, 0x2e, 0xf8, - 0x76, 0x89, 0x71, 0x68, 0x28, 0x12, 0xa4, 0x15, 0xaa, 0xc0, 0x0d, 0xbd, 0xce, 0xbd, 0x2e, 0xb3, - 0x6e, 0x06, 0x15, 0x3e, 0x34, 0xbd, 0x64, 0xa5, 0xd9, 0x00, 0xa2, 0x76, 0x07, 0x40, 0x78, 0x97, - 0x81, 0xa8, 0x9a, 0xf0, 0x4f, 0x17, 0x5a, 0x1f, 0x8a, 0x93, 0x2a, 0x64, 0x6f, 0x6b, 0x02, 0xec, - 0x25, 0x6c, 0x4b, 0x4d, 0x86, 0xc4, 0xda, 0x0d, 0x24, 0x0e, 0x9c, 0x64, 0xa9, 0x63, 0x2d, 0xd8, - 0xc2, 0xb2, 0x94, 0xa5, 0x8d, 0x30, 0x70, 0x12, 0x5b, 0xf6, 0x1a, 0xe0, 0x97, 0xa8, 0xf4, 0x29, - 0x45, 0x0f, 0x60, 0x7f, 0xe3, 0xde, 0x16, 0xcd, 0xee, 0x8f, 0x1a, 0x34, 0x57, 0xab, 0xec, 0x3d, - 0xf8, 0x96, 0x61, 0x16, 0x5e, 0x8c, 0xb7, 0xfa, 0x2f, 0x6d, 0x3f, 0xbd, 0x41, 0x61, 0xcd, 0x23, - 0x87, 0xbd, 0x02, 0xaf, 0x8f, 0xc4, 0x1e, 0x5f, 0x68, 0x2b, 0xf8, 0x6d, 0xef, 0xad, 0x4f, 0x7f, - 0xd5, 0x8d, 0x1c, 0x76, 0x0c, 0xbe, 0x85, 0x6b, 0xfd, 0x3e, 0xd5, 0xb8, 0x5d, 0x6b, 0xf2, 0xc2, - 0x9d, 0xc7, 0xb2, 0xf9, 0xd7, 0x6d, 0xaa, 0x27, 0xb9, 0x1e, 0xeb, 0x9a, 0x37, 0x8b, 0x9c, 0xde, - 0xd6, 0x27, 0x4f, 0x14, 0xd9, 0xc8, 0x37, 0x53, 0x39, 0xfa, 0x17, 0x00, 0x00, 0xff, 0xff, 0x9d, - 0xc6, 0x54, 0x3a, 0x55, 0x05, 0x00, 0x00, + // 641 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0xcf, 0x6e, 0xd3, 0x4e, + 0x10, 0x8e, 0xf3, 0xc7, 0x4d, 0xa6, 0xd5, 0xef, 0xb0, 0x3f, 0x9a, 0x9a, 0x00, 0xad, 0xf1, 0x85, + 0x48, 0xa8, 0x36, 0xa4, 0xe2, 0x88, 0x90, 0x0a, 0xa5, 0xad, 0x40, 0x20, 0x1c, 0x55, 0x48, 0x1c, + 0x90, 0x36, 0xe9, 0xd4, 0xb1, 0x68, 0xbd, 0x66, 0x77, 0x8d, 0xe8, 0x9b, 0xf0, 0x24, 0x1c, 0x39, + 0xf3, 0x0c, 0x1c, 0xfa, 0x14, 0x9c, 0x11, 0xf2, 0xae, 0xed, 0xb8, 0xb1, 0xdb, 0x43, 0x5b, 0x6e, + 0x1e, 0xcf, 0xcc, 0xb7, 0xf3, 0xe7, 0xfb, 0x06, 0xee, 0xc6, 0x9c, 0x49, 0x36, 0x49, 0x8e, 0x3c, + 0x1a, 0x87, 0x1e, 0x7e, 0xc5, 0x69, 0x22, 0x43, 0x16, 0xb9, 0xea, 0x37, 0xe9, 0x9e, 0xa0, 0x08, + 0x5c, 0x1a, 0x87, 0x03, 0x27, 0x60, 0x01, 0xf3, 0x8a, 0xe0, 0xd4, 0x52, 0x86, 0xfa, 0xd2, 0xd1, + 0x83, 0x3b, 0x85, 0x5b, 0x9e, 0xc6, 0x28, 0x3c, 0x21, 0x79, 0x32, 0x95, 0x99, 0x73, 0x7d, 0xc1, + 0xb9, 0xf0, 0x94, 0xf3, 0xa3, 0x0d, 0xfd, 0xe7, 0x1c, 0xa9, 0xc4, 0x9d, 0xdc, 0xe3, 0xe3, 0xe7, + 0x04, 0x85, 0x24, 0xef, 0x61, 0x25, 0x8c, 0x84, 0xa4, 0xd1, 0x14, 0xf7, 0xa8, 0x98, 0x59, 0x86, + 0x6d, 0x0c, 0x57, 0xb6, 0xb7, 0x7e, 0x9e, 0x6d, 0x34, 0x7e, 0x9d, 0x6d, 0x3c, 0x0c, 0x42, 0x39, + 0x4b, 0x26, 0xee, 0x94, 0x9d, 0x78, 0x69, 0xb9, 0x9b, 0x47, 0x2c, 0x89, 0x0e, 0x69, 0x0a, 0xe0, + 0x61, 0x14, 0x84, 0x11, 0x7a, 0x33, 0x2a, 0x66, 0x6e, 0x9a, 0xea, 0x9f, 0x03, 0x22, 0x16, 0x2c, + 0x49, 0x2a, 0x3e, 0xbd, 0xc2, 0x53, 0xab, 0x69, 0x1b, 0xc3, 0x9e, 0x9f, 0x9b, 0x64, 0x13, 0xcc, + 0x30, 0x8a, 0x13, 0x29, 0xac, 0x96, 0x6d, 0x0c, 0x97, 0x47, 0xab, 0xae, 0x9a, 0x44, 0xde, 0x83, + 0x3b, 0x56, 0xad, 0xf9, 0x59, 0x10, 0x21, 0xd0, 0x96, 0x34, 0x10, 0x56, 0xdb, 0x6e, 0x0d, 0x7b, + 0xbe, 0xfa, 0x26, 0x63, 0x80, 0x98, 0x72, 0x8c, 0xa4, 0xaa, 0xb9, 0x73, 0xf5, 0x9a, 0x4b, 0x30, + 0xe4, 0x1d, 0xf4, 0xf0, 0x4b, 0x8e, 0x69, 0x5e, 0x1d, 0x73, 0x8e, 0x42, 0x0e, 0x60, 0x39, 0xe6, + 0x6c, 0x8a, 0x42, 0x28, 0xd0, 0xa5, 0xab, 0x83, 0x96, 0x71, 0x48, 0x1f, 0x4c, 0x21, 0x31, 0xde, + 0x7f, 0x61, 0x75, 0xd5, 0x68, 0x33, 0x2b, 0x5d, 0xa6, 0x5e, 0x3d, 0xe3, 0xea, 0xbd, 0xde, 0x35, + 0x96, 0x59, 0x06, 0x72, 0x26, 0xb0, 0x56, 0xe1, 0x8f, 0x88, 0x59, 0x24, 0x90, 0xec, 0x42, 0x7b, + 0x76, 0x4d, 0xe2, 0x28, 0x00, 0xe7, 0x23, 0xfc, 0xbf, 0x8b, 0xb2, 0x42, 0xd0, 0x1b, 0xc3, 0xff, + 0xdd, 0x84, 0xfe, 0x58, 0x72, 0xa4, 0x27, 0x95, 0x37, 0x9e, 0x81, 0x79, 0x14, 0x1e, 0x4b, 0xe4, + 0xea, 0x95, 0xe5, 0xd1, 0x03, 0x37, 0xd7, 0xa6, 0x5b, 0x9f, 0xe1, 0xbe, 0x54, 0xe1, 0x7e, 0x96, + 0x36, 0xf8, 0xd6, 0x04, 0x53, 0xff, 0x22, 0x2e, 0x74, 0x85, 0xa4, 0x32, 0x11, 0x28, 0x2c, 0xc3, + 0x6e, 0x0d, 0xff, 0x1b, 0x11, 0x8d, 0xa6, 0xa4, 0xe9, 0x8e, 0x95, 0xcf, 0x2f, 0x62, 0x2a, 0x02, + 0x6c, 0xfe, 0x03, 0x01, 0xb6, 0xce, 0x0b, 0xb0, 0x4e, 0x51, 0x8b, 0xd4, 0xe9, 0xdc, 0x14, 0x75, + 0xbe, 0x1b, 0xd0, 0x3f, 0x88, 0x0f, 0xeb, 0x6e, 0xcf, 0x4d, 0xad, 0x96, 0x3c, 0x86, 0x25, 0x96, + 0x48, 0x75, 0x52, 0x9a, 0x97, 0x9c, 0x94, 0xbd, 0x86, 0x9f, 0xc7, 0x91, 0x3e, 0x74, 0x90, 0x73, + 0xc6, 0xf5, 0x6c, 0xf6, 0x1a, 0xbe, 0x36, 0xb7, 0xbb, 0x60, 0x72, 0x14, 0xc9, 0xb1, 0x74, 0x6e, + 0xc3, 0x5a, 0xa5, 0x6e, 0xcd, 0x79, 0xa7, 0x0f, 0xb7, 0x5e, 0x87, 0xa2, 0xc2, 0x55, 0xe7, 0x0d, + 0xac, 0x2e, 0xfc, 0xcf, 0x44, 0xf2, 0x04, 0xa0, 0xb8, 0xc9, 0x9a, 0x16, 0x45, 0x8d, 0x9a, 0x16, + 0xf3, 0x94, 0x52, 0xe0, 0xe8, 0x4f, 0x13, 0x7a, 0x85, 0x87, 0xbc, 0x05, 0x53, 0x8b, 0x90, 0xd8, + 0x73, 0x7e, 0xd6, 0x9f, 0xf5, 0xc1, 0xfd, 0x4b, 0x22, 0xb2, 0x26, 0x1a, 0xe4, 0x29, 0xb4, 0x76, + 0x51, 0x92, 0x7b, 0xf3, 0xd8, 0x1a, 0x01, 0x0e, 0xea, 0xeb, 0x74, 0x1a, 0x64, 0x1f, 0xda, 0x69, + 0xb7, 0x64, 0x7d, 0x9e, 0x5f, 0x37, 0x95, 0xc1, 0xc6, 0x85, 0xfe, 0xa2, 0x92, 0x1d, 0x30, 0xb5, + 0xd0, 0xca, 0xad, 0xd5, 0x4b, 0xef, 0xc2, 0x7a, 0x1e, 0x19, 0xe9, 0x84, 0xf4, 0xca, 0xca, 0x30, + 0xf5, 0xe4, 0x2b, 0x4f, 0xe8, 0xa2, 0x35, 0x37, 0xb6, 0x3b, 0x1f, 0x5a, 0x34, 0x0e, 0x27, 0xa6, + 0x22, 0xd2, 0xd6, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x08, 0x34, 0x0c, 0x6c, 0xd1, 0x07, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -428,6 +523,8 @@ type ExecutionClient interface { Create(ctx context.Context, in *CreateExecutionRequest, opts ...grpc.CallOption) (*CreateExecutionResponse, error) // Get returns a single Execution specified in a request. Get(ctx context.Context, in *GetExecutionRequest, opts ...grpc.CallOption) (*execution.Execution, error) + // List returns all Executions matching the criteria of the request. + List(ctx context.Context, in *ListExecutionRequest, opts ...grpc.CallOption) (*ListExecutionResponse, error) // Stream returns a stream of executions that satisfy criteria // specified in a request. Stream(ctx context.Context, in *StreamExecutionRequest, opts ...grpc.CallOption) (Execution_StreamClient, error) @@ -461,6 +558,15 @@ func (c *executionClient) Get(ctx context.Context, in *GetExecutionRequest, opts return out, nil } +func (c *executionClient) List(ctx context.Context, in *ListExecutionRequest, opts ...grpc.CallOption) (*ListExecutionResponse, error) { + out := new(ListExecutionResponse) + err := c.cc.Invoke(ctx, "/mesg.api.Execution/List", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *executionClient) Stream(ctx context.Context, in *StreamExecutionRequest, opts ...grpc.CallOption) (Execution_StreamClient, error) { stream, err := c.cc.NewStream(ctx, &_Execution_serviceDesc.Streams[0], "/mesg.api.Execution/Stream", opts...) if err != nil { @@ -508,6 +614,8 @@ type ExecutionServer interface { Create(context.Context, *CreateExecutionRequest) (*CreateExecutionResponse, error) // Get returns a single Execution specified in a request. Get(context.Context, *GetExecutionRequest) (*execution.Execution, error) + // List returns all Executions matching the criteria of the request. + List(context.Context, *ListExecutionRequest) (*ListExecutionResponse, error) // Stream returns a stream of executions that satisfy criteria // specified in a request. Stream(*StreamExecutionRequest, Execution_StreamServer) error @@ -525,6 +633,9 @@ func (*UnimplementedExecutionServer) Create(ctx context.Context, req *CreateExec func (*UnimplementedExecutionServer) Get(ctx context.Context, req *GetExecutionRequest) (*execution.Execution, error) { return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") } +func (*UnimplementedExecutionServer) List(ctx context.Context, req *ListExecutionRequest) (*ListExecutionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method List not implemented") +} func (*UnimplementedExecutionServer) Stream(req *StreamExecutionRequest, srv Execution_StreamServer) error { return status.Errorf(codes.Unimplemented, "method Stream not implemented") } @@ -572,6 +683,24 @@ func _Execution_Get_Handler(srv interface{}, ctx context.Context, dec func(inter return interceptor(ctx, in, info, handler) } +func _Execution_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListExecutionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionServer).List(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/mesg.api.Execution/List", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionServer).List(ctx, req.(*ListExecutionRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Execution_Stream_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(StreamExecutionRequest) if err := stream.RecvMsg(m); err != nil { @@ -623,6 +752,10 @@ var _Execution_serviceDesc = grpc.ServiceDesc{ MethodName: "Get", Handler: _Execution_Get_Handler, }, + { + MethodName: "List", + Handler: _Execution_List_Handler, + }, { MethodName: "Update", Handler: _Execution_Update_Handler, diff --git a/protobuf/api/execution.proto b/protobuf/api/execution.proto index e99dbdfd6..b9d9dee73 100644 --- a/protobuf/api/execution.proto +++ b/protobuf/api/execution.proto @@ -20,6 +20,9 @@ service Execution { // Get returns a single Execution specified in a request. rpc Get(GetExecutionRequest) returns (types.Execution) {} + // List returns all Executions matching the criteria of the request. + rpc List(ListExecutionRequest) returns (ListExecutionResponse) {} + // Stream returns a stream of executions that satisfy criteria // specified in a request. rpc Stream(StreamExecutionRequest) returns (stream types.Execution) {} @@ -37,6 +40,23 @@ message CreateExecutionRequest { string taskKey = 2; mesg.protobuf.Struct inputs = 3; repeated string tags = 4; + bytes parentHash = 5 [ + (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", + (gogoproto.nullable) = false + ]; + bytes eventHash = 6 [ + (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", + (gogoproto.nullable) = false + ]; + bytes processHash = 7 [ + (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", + (gogoproto.nullable) = false + ]; + string stepID = 8; + bytes executorHash = 9 [ + (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", + (gogoproto.nullable) = false + ]; } // CreateExecutionResponse defines response for execution creation. @@ -75,6 +95,12 @@ message StreamExecutionRequest{ // tags to filter executions. All tags needs to be present in the execution. repeated string tags = 4; + + // Executor's hash to filter executions. + bytes executorHash = 5 [ + (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", + (gogoproto.nullable) = false + ]; } // Filter used to filter a stream of executions. @@ -101,3 +127,12 @@ message UpdateExecutionRequest { // UpdateExecutionResponse defines response for execution update. message UpdateExecutionResponse {} + +// The request's data for the `List` API. +message ListExecutionRequest {} + +// The response's data for the `List` API. +message ListExecutionResponse { + // List of executions that match the request's filters. + repeated types.Execution executions = 1; +} diff --git a/scripts/build-mocks.sh b/scripts/build-mocks.sh index 28ecf2e37..5dbd5bed8 100755 --- a/scripts/build-mocks.sh +++ b/scripts/build-mocks.sh @@ -4,3 +4,4 @@ mockery -name ExecutionSDK -dir ./orchestrator -output ./orchestrator/mocks mockery -name EventSDK -dir ./orchestrator -output ./orchestrator/mocks mockery -name ProcessSDK -dir ./orchestrator -output ./orchestrator/mocks +mockery -name RunnerSDK -dir ./orchestrator -output ./orchestrator/mocks diff --git a/sdk/backend.go b/sdk/backend.go index b2ae09c8e..d9fe2aa0b 100644 --- a/sdk/backend.go +++ b/sdk/backend.go @@ -13,6 +13,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/supply" "github.com/mesg-foundation/engine/codec" "github.com/mesg-foundation/engine/cosmos" + executionsdk "github.com/mesg-foundation/engine/sdk/execution" instancesdk "github.com/mesg-foundation/engine/sdk/instance" ownershipsdk "github.com/mesg-foundation/engine/sdk/ownership" runnersdk "github.com/mesg-foundation/engine/sdk/runner" @@ -22,6 +23,7 @@ import ( // Backend handles all the backend functions. type Backend struct { Service *servicesdk.Backend + Execution *executionsdk.Backend Ownership *ownershipsdk.Backend Instance *instancesdk.Backend Runner *runnersdk.Backend @@ -34,11 +36,13 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { service := servicesdk.NewBackend(appFactory, ownership) instance := instancesdk.NewBackend(appFactory) runner := runnersdk.NewBackend(appFactory, instance) + execution := executionsdk.NewBackend(appFactory, service, instance) return &Backend{ Service: service, Ownership: ownership, Instance: instance, Runner: runner, + Execution: execution, } } diff --git a/sdk/sdk.go b/sdk/sdk.go index 0882d7528..57a7aac86 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -19,7 +19,7 @@ import ( type SDK struct { Service *servicesdk.SDK Instance *instancesdk.SDK - Execution *executionsdk.Execution + Execution *executionsdk.SDK Event *eventsdk.Event Process *processesdk.Process Account *accountsdk.SDK @@ -28,7 +28,7 @@ type SDK struct { } // New creates a new SDK with given options. -func New(client *cosmos.Client, kb *cosmos.Keybase, execDB database.ExecutionDB, processDB database.ProcessDB, container container.Container, engineName, port string, ipfsEndpoint string) *SDK { +func New(client *cosmos.Client, kb *cosmos.Keybase, processDB database.ProcessDB, container container.Container, engineName, port string, ipfsEndpoint string) *SDK { ps := pubsub.New(0) accountSDK := accountsdk.NewSDK(kb) serviceSDK := servicesdk.New(client, accountSDK) @@ -36,7 +36,7 @@ func New(client *cosmos.Client, kb *cosmos.Keybase, execDB database.ExecutionDB, instanceSDK := instancesdk.New(client) runnerSDK := runnersdk.New(client, accountSDK, serviceSDK, instanceSDK, container, engineName, port, ipfsEndpoint) processSDK := processesdk.New(instanceSDK, processDB) - executionSDK := executionsdk.New(ps, serviceSDK, instanceSDK, processSDK, execDB) + executionSDK := executionsdk.New(client, accountSDK, serviceSDK, instanceSDK, runnerSDK) eventSDK := eventsdk.New(ps, serviceSDK, instanceSDK) return &SDK{ Service: serviceSDK, diff --git a/server/grpc/api/execution.go b/server/grpc/api/execution.go index 991862c6c..42f324230 100644 --- a/server/grpc/api/execution.go +++ b/server/grpc/api/execution.go @@ -2,19 +2,13 @@ package api import ( "context" - "errors" "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/protobuf/acknowledgement" "github.com/mesg-foundation/engine/protobuf/api" "github.com/mesg-foundation/engine/sdk" - executionsdk "github.com/mesg-foundation/engine/sdk/execution" ) -// ErrNoOutput is an error when there is no output for updating execution. -var ErrNoOutput = errors.New("output not supplied") - // ExecutionServer serve execution functions. type ExecutionServer struct { sdk *sdk.SDK @@ -27,17 +21,17 @@ func NewExecutionServer(sdk *sdk.SDK) *ExecutionServer { // Create creates an execution. func (s *ExecutionServer) Create(ctx context.Context, req *api.CreateExecutionRequest) (*api.CreateExecutionResponse, error) { - eventHash, err := hash.Random() + credUsername, credPassphrase, err := GetCredentialFromContext(ctx) if err != nil { return nil, err } - executionHash, err := s.sdk.Execution.Execute(nil, req.InstanceHash, eventHash, nil, "", req.TaskKey, req.Inputs, req.Tags) + exec, err := s.sdk.Execution.Create(req, credUsername, credPassphrase) if err != nil { return nil, err } return &api.CreateExecutionResponse{ - Hash: executionHash, + Hash: exec.Hash, }, nil } @@ -48,48 +42,45 @@ func (s *ExecutionServer) Get(ctx context.Context, req *api.GetExecutionRequest) // Stream returns stream of executions. func (s *ExecutionServer) Stream(req *api.StreamExecutionRequest, resp api.Execution_StreamServer) error { - var f *executionsdk.Filter - - if req.Filter != nil { - f = &executionsdk.Filter{ - InstanceHash: req.Filter.InstanceHash, - Statuses: req.Filter.Statuses, - Tags: req.Filter.Tags, - TaskKey: req.Filter.TaskKey, + stream, closer, err := s.sdk.Execution.Stream(req) + defer func() { + err := closer() + if err != nil { + // TODO: remove panic + panic(err) } + }() + if err != nil { + return err } - - stream := s.sdk.Execution.GetStream(f) - defer stream.Close() - - // send header to notify client that the stream is ready. if err := acknowledgement.SetStreamReady(resp); err != nil { return err } - - for exec := range stream.C { + for exec := range stream { if err := resp.Send(exec); err != nil { return err } } - return nil } // Update updates execution from given hash. func (s *ExecutionServer) Update(ctx context.Context, req *api.UpdateExecutionRequest) (*api.UpdateExecutionResponse, error) { - var err error - switch res := req.Result.(type) { - case *api.UpdateExecutionRequest_Outputs: - err = s.sdk.Execution.Update(req.Hash, res.Outputs, nil) - case *api.UpdateExecutionRequest_Error: - err = s.sdk.Execution.Update(req.Hash, nil, errors.New(res.Error)) - default: - err = ErrNoOutput + credUsername, credPassphrase, err := GetCredentialFromContext(ctx) + if err != nil { + return nil, err + } + if _, err := s.sdk.Execution.Update(req, credUsername, credPassphrase); err != nil { + return nil, err } + return &api.UpdateExecutionResponse{}, nil +} +// List returns all executions. +func (s *ExecutionServer) List(ctx context.Context, req *api.ListExecutionRequest) (*api.ListExecutionResponse, error) { + executions, err := s.sdk.Execution.List() if err != nil { return nil, err } - return &api.UpdateExecutionResponse{}, nil + return &api.ListExecutionResponse{Executions: executions}, nil } diff --git a/server/grpc/api/execution_test.go b/server/grpc/api/execution_test.go deleted file mode 100644 index 26d956c56..000000000 --- a/server/grpc/api/execution_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package api - -import ( - "context" - "os" - "testing" - - "github.com/mesg-foundation/engine/database" - "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/protobuf/api" - "github.com/mesg-foundation/engine/sdk" - "github.com/stretchr/testify/require" -) - -const execdbname = "exec.db.test" - -func TestGet(t *testing.T) { - db, err := database.NewExecutionDB(execdbname) - require.NoError(t, err) - defer db.Close() - defer os.RemoveAll(execdbname) - - exec := execution.New(nil, nil, nil, nil, "", "", nil, nil) - require.NoError(t, db.Save(exec)) - - sdk := sdk.New(nil, nil, db, nil, nil, "", "", "") - s := NewExecutionServer(sdk) - - got, err := s.Get(context.Background(), &api.GetExecutionRequest{Hash: exec.Hash}) - require.NoError(t, err) - require.True(t, got.Equal(exec)) -} - -func TestUpdate(t *testing.T) { - db, err := database.NewExecutionDB(execdbname) - require.NoError(t, err) - defer db.Close() - defer os.RemoveAll(execdbname) - - exec := execution.New(nil, nil, nil, nil, "", "", nil, nil) - require.NoError(t, db.Save(exec)) - - sdk := sdk.New(nil, nil, db, nil, nil, "", "", "") - s := NewExecutionServer(sdk) - - _, err = s.Update(context.Background(), &api.UpdateExecutionRequest{Hash: exec.Hash}) - require.Equal(t, ErrNoOutput, err) -} From 3216db2fdeb9707134805cd8c9da6dd6c0be28ef Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 17:25:02 +0700 Subject: [PATCH 07/19] Remove useless files from database package --- database/execution_db.go | 120 -------------------------- database/execution_db_test.go | 67 --------------- database/leveldb.go | 18 ---- database/store/cosmos.go | 152 --------------------------------- database/store/cosmos_test.go | 72 ---------------- database/store/interface.go | 40 --------- database/store/leveldb.go | 51 ----------- database/store/leveldb_test.go | 76 ----------------- 8 files changed, 596 deletions(-) delete mode 100644 database/execution_db.go delete mode 100644 database/execution_db_test.go delete mode 100644 database/leveldb.go delete mode 100644 database/store/cosmos.go delete mode 100644 database/store/cosmos_test.go delete mode 100644 database/store/interface.go delete mode 100644 database/store/leveldb.go delete mode 100644 database/store/leveldb_test.go diff --git a/database/execution_db.go b/database/execution_db.go deleted file mode 100644 index bd1d02661..000000000 --- a/database/execution_db.go +++ /dev/null @@ -1,120 +0,0 @@ -package database - -import ( - "errors" - "io" - - "github.com/gogo/protobuf/proto" - "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/hash" - "github.com/syndtr/goleveldb/leveldb" -) - -// ExecutionDB exposes all the functionalities -type ExecutionDB interface { - Find(executionHash hash.Hash) (*execution.Execution, error) - Save(execution *execution.Execution) error - OpenTransaction() (ExecutionTransaction, error) - io.Closer -} - -// ExecutionTransaction is the transaction handle. -type ExecutionTransaction interface { - Find(executionHash hash.Hash) (*execution.Execution, error) - Save(execution *execution.Execution) error - Commit() error - Discard() -} - -// LevelDBExecutionDB is a concrete implementation of the DB interface -type LevelDBExecutionDB struct { - db *leveldb.DB -} - -// NewExecutionDB creates a new DB instance -func NewExecutionDB(path string) (*LevelDBExecutionDB, error) { - db, err := leveldb.OpenFile(path, nil) - if err != nil { - return nil, err - } - - return &LevelDBExecutionDB{db: db}, nil -} - -// Find the execution based on an executionHash, returns an error if not found -func (db *LevelDBExecutionDB) Find(executionHash hash.Hash) (*execution.Execution, error) { - return executionFind(db.db, executionHash) -} - -// Save an instance of executable in the database -// Returns an error if anything from marshaling to database saving goes wrong -func (db *LevelDBExecutionDB) Save(execution *execution.Execution) error { - return executionSave(db.db, execution) -} - -// OpenTransaction opens an atomic DB transaction. Only one transaction can be -// opened at a time. -func (db *LevelDBExecutionDB) OpenTransaction() (ExecutionTransaction, error) { - tx, err := db.db.OpenTransaction() - if err != nil { - return nil, err - } - return &LevelDBExecutionTransaction{tx: tx}, nil -} - -// Close closes database. -func (db *LevelDBExecutionDB) Close() error { - return db.db.Close() -} - -// LevelDBExecutionTransaction is the transaction handle. -type LevelDBExecutionTransaction struct { - tx *leveldb.Transaction -} - -// Find the execution based on an executionHash, returns an error if not found -func (tx *LevelDBExecutionTransaction) Find(executionHash hash.Hash) (*execution.Execution, error) { - return executionFind(tx.tx, executionHash) -} - -// Save an instance of executable in the database -// Returns an error if anything from marshaling to database saving goes wrong -func (tx *LevelDBExecutionTransaction) Save(execution *execution.Execution) error { - return executionSave(tx.tx, execution) -} - -// Commit commits the transaction. -func (tx *LevelDBExecutionTransaction) Commit() error { - return tx.tx.Commit() -} - -// Discard discards the transaction. -func (tx *LevelDBExecutionTransaction) Discard() { - tx.tx.Discard() -} - -// Find the execution based on an executionHash, returns an error if not found -func executionFind(db leveldbTxDB, executionHash hash.Hash) (*execution.Execution, error) { - data, err := db.Get(executionHash, nil) - if err != nil { - return nil, err - } - var execution execution.Execution - if err := proto.Unmarshal(data, &execution); err != nil { - return nil, err - } - return &execution, nil -} - -// Save an instance of executable in the database -// Returns an error if anything from marshaling to database saving goes wrong -func executionSave(db leveldbTxDB, execution *execution.Execution) error { - if len(execution.Hash) == 0 { - return errors.New("database: can't save execution without hash") - } - data, err := proto.Marshal(execution) - if err != nil { - return err - } - return db.Put(execution.Hash, data, nil) -} diff --git a/database/execution_db_test.go b/database/execution_db_test.go deleted file mode 100644 index 6141f1bae..000000000 --- a/database/execution_db_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package database - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/mesg-foundation/engine/execution" - "github.com/mesg-foundation/engine/hash" - "github.com/stretchr/testify/require" -) - -func db(t *testing.T, dir string) ExecutionDB { - db, err := NewExecutionDB(dir) - require.NoError(t, err) - return db -} - -func TestFind(t *testing.T) { - dir, _ := ioutil.TempDir("", "TestFind") - defer os.RemoveAll(dir) - db := db(t, dir) - defer db.Close() - e := &execution.Execution{Hash: hash.Int(1)} - db.Save(e) - tests := []struct { - hash hash.Hash - hasError bool - }{ - {hash: e.Hash, hasError: false}, - {hash: hash.Int(2), hasError: true}, - } - for _, test := range tests { - execution, err := db.Find(test.hash) - if test.hasError { - require.Error(t, err) - continue - } - require.NoError(t, err) - require.NotNil(t, execution) - e, err := db.Find(execution.Hash) - require.NoError(t, err) - require.NotNil(t, e) - } -} - -func TestSave(t *testing.T) { - dir, _ := ioutil.TempDir("", "TestSave") - defer os.RemoveAll(dir) - db := db(t, dir) - defer db.Close() - tests := []struct { - execution *execution.Execution - hasError bool - }{ - {&execution.Execution{Hash: hash.Int(1)}, false}, - {&execution.Execution{}, true}, - } - for _, test := range tests { - err := db.Save(test.execution) - if test.hasError { - require.Error(t, err) - continue - } - require.NoError(t, err) - } -} diff --git a/database/leveldb.go b/database/leveldb.go deleted file mode 100644 index cc26084d0..000000000 --- a/database/leveldb.go +++ /dev/null @@ -1,18 +0,0 @@ -package database - -import ( - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// txdb is common interface for leveldb database and transaction. -type leveldbTxDB interface { - Delete(key []byte, wo *opt.WriteOptions) error - Get(key []byte, ro *opt.ReadOptions) ([]byte, error) - Has(key []byte, ro *opt.ReadOptions) (bool, error) - NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator - Put(key, value []byte, wo *opt.WriteOptions) error - Write(b *leveldb.Batch, wo *opt.WriteOptions) error -} diff --git a/database/store/cosmos.go b/database/store/cosmos.go deleted file mode 100644 index f786f6a10..000000000 --- a/database/store/cosmos.go +++ /dev/null @@ -1,152 +0,0 @@ -package store - -import ( - "errors" - "fmt" - - "github.com/cosmos/cosmos-sdk/types" -) - -// CosmosStore is a Cosmos KVStore implementation of Store. -type CosmosStore struct { - store types.KVStore -} - -// NewCosmosStore returns a new Cosmos KVStore wrapper. -func NewCosmosStore(store types.KVStore) *CosmosStore { - return &CosmosStore{ - store: store, - } -} - -// NewIterator returns a new iterator. -func (s *CosmosStore) NewIterator() Iterator { - return NewCosmosIterator(types.KVStorePrefixIterator(s.store, nil)) -} - -// Has returns true if the key is set in the store. -func (s *CosmosStore) Has(key []byte) (has bool, err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - if err, ok = r.(error); !ok { - err = fmt.Errorf("store: %s", r) - } - } - }() - has = s.store.Has(key) - return -} - -// Get retrives service from store. It returns an error if the store does not contains the key. -func (s *CosmosStore) Get(key []byte) (out []byte, err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - if err, ok = r.(error); !ok { - err = fmt.Errorf("store: %s", r) - } - } - }() - - has, err := s.Has(key) - if err != nil { - return nil, err - } - if !has { - return nil, errors.New("not found") - } - out = s.store.Get(key) - return -} - -// Delete deletes the value for the given key. Delete will not returns error if key doesn't exist. -func (s *CosmosStore) Delete(key []byte) (err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - if err, ok = r.(error); !ok { - err = fmt.Errorf("store: %s", r) - } - } - }() - s.store.Delete(key) - return -} - -// Put sets the value for the given key. It overwrites any previous value. -func (s *CosmosStore) Put(key []byte, value []byte) (err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - if err, ok = r.(error); !ok { - err = fmt.Errorf("store: %s", r) - } - } - }() - s.store.Set(key, value) - return -} - -// Close closes the store. -func (s *CosmosStore) Close() error { - return nil -} - -// CosmosIterator is a Cosmos KVStore's iterator implementation of Iterator. -type CosmosIterator struct { - iter types.Iterator - err error - valid bool // HACK for next function. Iterator of cosmos already contains the first element at its creation. -} - -// NewCosmosIterator returns a new Cosmos KVStore Iterator wrapper. -func NewCosmosIterator(iter types.Iterator) *CosmosIterator { - return &CosmosIterator{ - iter: iter, - valid: false, - } -} - -// Next moves the iterator to the next sequential key in the store. -func (i *CosmosIterator) Next() bool { - defer i.handleError() - if i.valid { - i.iter.Next() - } - i.valid = i.iter.Valid() - return i.valid -} - -// Key returns the key of the cursor. -func (i *CosmosIterator) Key() []byte { - defer i.handleError() - return i.iter.Key() -} - -// Value returns the value of the cursor. -func (i *CosmosIterator) Value() []byte { - defer i.handleError() - return i.iter.Value() -} - -// Release releases the Iterator. -func (i *CosmosIterator) Release() { - defer i.handleError() - i.iter.Close() -} - -// Error returns any accumulated error. -func (i *CosmosIterator) Error() error { - return i.err -} - -// returns any accumulated error. -func (i *CosmosIterator) handleError() { - if r := recover(); r != nil { - var ok bool - if i.err, ok = r.(error); !ok { - i.err = fmt.Errorf("store iterator: %s", r) - } - } -} diff --git a/database/store/cosmos_test.go b/database/store/cosmos_test.go deleted file mode 100644 index 62fc7db89..000000000 --- a/database/store/cosmos_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package store - -import ( - "testing" - - "github.com/cosmos/cosmos-sdk/store/transient" - "github.com/stretchr/testify/require" -) - -func newCosmosStore(t *testing.T) (*CosmosStore, func()) { - store := NewCosmosStore(transient.NewStore()) - return store, func() { - require.NoError(t, store.Close()) - } -} - -func TestCosmosStorePut(t *testing.T) { - store, closer := newCosmosStore(t) - defer closer() - require.NoError(t, store.Put([]byte("hello"), []byte("world"))) -} - -func TestCosmosStoreDelete(t *testing.T) { - store, closer := newCosmosStore(t) - defer closer() - store.Put([]byte("hello"), []byte("world")) - require.NoError(t, store.Delete([]byte("hello"))) -} - -func TestCosmosStoreGet(t *testing.T) { - store, closer := newCosmosStore(t) - defer closer() - store.Put([]byte("hello"), []byte("world")) - value, err := store.Get([]byte("hello")) - require.NoError(t, err) - require.Equal(t, []byte("world"), value) -} - -func TestCosmosStoreHas(t *testing.T) { - store, closer := newCosmosStore(t) - defer closer() - store.Put([]byte("hello"), []byte("world")) - has, err := store.Has([]byte("hello")) - require.NoError(t, err) - require.True(t, has) -} - -func TestCosmosStoreIterate(t *testing.T) { - store, closer := newCosmosStore(t) - defer closer() - - data := []struct { - key []byte - value []byte - }{ - {key: []byte("hello"), value: []byte("world")}, - {key: []byte("foo"), value: []byte("bar")}, - } - for _, d := range data { - store.Put(d.key, d.value) - } - iter := store.NewIterator() - i := len(data) - 1 - for iter.Next() { - require.Equal(t, data[i].key, iter.Key()) - require.Equal(t, data[i].value, iter.Value()) - i-- - } - require.Equal(t, -1, i) - iter.Release() - require.NoError(t, iter.Error()) -} diff --git a/database/store/interface.go b/database/store/interface.go deleted file mode 100644 index 2f8d0883e..000000000 --- a/database/store/interface.go +++ /dev/null @@ -1,40 +0,0 @@ -package store - -// Store describes the public API of a store. -type Store interface { - // Get retrives service from store. It returns an error if the store does not contains the key. - Get(key []byte) ([]byte, error) - - // Has returns true if the key is set in the store. - Has(key []byte) (bool, error) - - // Delete deletes the value for the given key. Delete will not returns error if key doesn't exist. - Delete(key []byte) error - - // Put sets the value for the given key. It overwrites any previous value. - Put(key []byte, value []byte) error - - // NewIterator returns a new iterator. - NewIterator() Iterator - - // Close closes the store. - Close() error -} - -// Iterator describes the public API of an iterator. -type Iterator interface { - // Next moves the iterator to the next sequential key in the store. - Next() bool - - // Key returns the key of the cursor. - Key() []byte - - // Value returns the value of the cursor. - Value() []byte - - // Release releases the Iterator. - Release() - - // Error returns any accumulated error. - Error() error -} diff --git a/database/store/leveldb.go b/database/store/leveldb.go deleted file mode 100644 index 363b147e8..000000000 --- a/database/store/leveldb.go +++ /dev/null @@ -1,51 +0,0 @@ -package store - -import ( - "github.com/syndtr/goleveldb/leveldb" -) - -// LevelDBStore is a levelDB implementation of Store. -type LevelDBStore struct { - db *leveldb.DB -} - -// NewLevelDBStore returns a new level db wrapper. -func NewLevelDBStore(path string) (*LevelDBStore, error) { - db, err := leveldb.OpenFile(path, nil) - if err != nil { - return nil, err - } - return &LevelDBStore{ - db: db, - }, nil -} - -// NewIterator returns a new iterator. -func (s *LevelDBStore) NewIterator() Iterator { - return s.db.NewIterator(nil, nil) -} - -// Has returns true if the key is set in the store. -func (s *LevelDBStore) Has(key []byte) (bool, error) { - return s.db.Has(key, nil) -} - -// Get retrives service from store. It returns error if the store does not contains the key. -func (s *LevelDBStore) Get(key []byte) ([]byte, error) { - return s.db.Get(key, nil) -} - -// Delete deletes the value for the given key. Delete will not returns error if key doesn't exist. -func (s *LevelDBStore) Delete(key []byte) error { - return s.db.Delete(key, nil) -} - -// Put sets the value for the given key. It overwrites any previous value. -func (s *LevelDBStore) Put(key []byte, value []byte) error { - return s.db.Put(key, value, nil) -} - -// Close closes the store. -func (s *LevelDBStore) Close() error { - return s.db.Close() -} diff --git a/database/store/leveldb_test.go b/database/store/leveldb_test.go deleted file mode 100644 index 96f93df55..000000000 --- a/database/store/leveldb_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package store - -import ( - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -const testLevelDBPath = "test" - -func newLevelDBStore(t *testing.T) (*LevelDBStore, func()) { - store, err := NewLevelDBStore(testLevelDBPath) - require.NoError(t, err) - return store, func() { - require.NoError(t, store.Close()) - os.RemoveAll(testLevelDBPath) - } -} - -func TestLevelDBStorePut(t *testing.T) { - store, closer := newLevelDBStore(t) - defer closer() - require.NoError(t, store.Put([]byte("hello"), []byte("world"))) -} - -func TestLevelDBStoreDelete(t *testing.T) { - store, closer := newLevelDBStore(t) - defer closer() - store.Put([]byte("hello"), []byte("world")) - require.NoError(t, store.Delete([]byte("hello"))) -} - -func TestLevelDBStoreGet(t *testing.T) { - store, closer := newLevelDBStore(t) - defer closer() - store.Put([]byte("hello"), []byte("world")) - value, err := store.Get([]byte("hello")) - require.NoError(t, err) - require.Equal(t, []byte("world"), value) -} - -func TestLevelDBStoreHas(t *testing.T) { - store, closer := newLevelDBStore(t) - defer closer() - store.Put([]byte("hello"), []byte("world")) - has, err := store.Has([]byte("hello")) - require.NoError(t, err) - require.True(t, has) -} - -func TestLevelDBStoreIterate(t *testing.T) { - store, closer := newLevelDBStore(t) - defer closer() - - data := []struct { - key []byte - value []byte - }{ - {key: []byte("hello"), value: []byte("world")}, - {key: []byte("foo"), value: []byte("bar")}, - } - for _, d := range data { - store.Put(d.key, d.value) - } - iter := store.NewIterator() - i := len(data) - 1 - for iter.Next() { - require.Equal(t, data[i].key, iter.Key()) - require.Equal(t, data[i].value, iter.Value()) - i-- - } - require.Equal(t, -1, i) - iter.Release() - require.NoError(t, iter.Error()) -} From 02e26b848ab0388a8957b359c1bcd8c24b11f88c Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Tue, 19 Nov 2019 17:27:06 +0700 Subject: [PATCH 08/19] Update e2e test service --- e2e/execution_test.go | 121 +++++++++++++++++++++--------- e2e/testdata/e2e.config.yml | 2 +- e2e/testdata/test-service.json | 2 +- e2e/testdata/test-service/go.mod | 2 +- e2e/testdata/test-service/go.sum | 58 ++++++++------ e2e/testdata/test-service/main.go | 2 +- go.mod | 2 +- 7 files changed, 125 insertions(+), 64 deletions(-) diff --git a/e2e/execution_test.go b/e2e/execution_test.go index 854e012eb..f524a0d5b 100644 --- a/e2e/execution_test.go +++ b/e2e/execution_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/mesg-foundation/engine/execution" + "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/protobuf/acknowledgement" pb "github.com/mesg-foundation/engine/protobuf/api" "github.com/mesg-foundation/engine/protobuf/types" @@ -13,47 +14,93 @@ import ( ) func testExecution(t *testing.T) { - stream, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ - Filter: &pb.StreamExecutionRequest_Filter{ - InstanceHash: testInstanceHash, - }, - }) - require.NoError(t, err) - acknowledgement.WaitForStreamToBeReady(stream) - + var executionHash hash.Hash + var execPingCompleted *execution.Execution ctx := metadata.NewOutgoingContext(context.Background(), passmd) - resp, err := client.ExecutionClient.Create(ctx, &pb.CreateExecutionRequest{ - InstanceHash: testInstanceHash, - TaskKey: "ping", - Inputs: &types.Struct{ - Fields: map[string]*types.Value{ - "msg": { - Kind: &types.Value_StringValue{ - StringValue: "test", + t.Run("stream", func(t *testing.T) { + t.Run("with nil filter", func(t *testing.T) { + _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{}) + require.NoError(t, err) + }) + // TODO: no error are returned but it supposed to... + // t.Run("with not valid filter", func(t *testing.T) { + // t.Run("not found executor", func(t *testing.T) { + // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ + // Filter: &pb.StreamExecutionRequest_Filter{ + // ExecutorHash: hash.Int(1), + // }, + // }) + // require.EqualError(t, err, "dwdw") + // }) + // t.Run("not found instance", func(t *testing.T) { + // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ + // Filter: &pb.StreamExecutionRequest_Filter{ + // InstanceHash: hash.Int(1), + // }, + // }) + // require.EqualError(t, err, "dwdw") + // }) + // t.Run("not found task key", func(t *testing.T) { + // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ + // Filter: &pb.StreamExecutionRequest_Filter{ + // ExecutorHash: testRunnerHash, + // TaskKey: "do-not-exist", + // }, + // }) + // require.EqualError(t, err, "service \"test-service\" - task \"do-not-exist\" not found") + // }) + // }) + t.Run("working", func(t *testing.T) { + stream, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ + Filter: &pb.StreamExecutionRequest_Filter{ + ExecutorHash: testRunnerHash, + }, + }) + require.NoError(t, err) + acknowledgement.WaitForStreamToBeReady(stream) + resp, err := client.ExecutionClient.Create(ctx, &pb.CreateExecutionRequest{ + InstanceHash: testInstanceHash, + TaskKey: "ping", + EventHash: hash.Int(1), + ExecutorHash: testRunnerHash, + Inputs: &types.Struct{ + Fields: map[string]*types.Value{ + "msg": { + Kind: &types.Value_StringValue{ + StringValue: "test", + }, + }, }, }, - }, - }, + }) + require.NoError(t, err) + executionHash = resp.Hash + t.Run("receive in progress execution", func(t *testing.T) { + execPingInProgress, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, resp.Hash, execPingInProgress.Hash) + require.Equal(t, "ping", execPingInProgress.TaskKey) + require.Equal(t, execution.Status_InProgress, execPingInProgress.Status) + }) + t.Run("receive completed execution", func(t *testing.T) { + execPingCompleted, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, resp.Hash, execPingCompleted.Hash) + require.Equal(t, "ping", execPingCompleted.TaskKey) + require.Equal(t, execution.Status_Completed, execPingCompleted.Status) + }) + }) }) - require.NoError(t, err) - // receive in progress status - exec, err := stream.Recv() - require.NoError(t, err) - require.Equal(t, resp.Hash, exec.Hash) - require.Equal(t, "ping", exec.TaskKey) - require.Equal(t, execution.Status_InProgress, exec.Status) - - // receive completed status - exec, err = stream.Recv() - require.NoError(t, err) - require.Equal(t, resp.Hash, exec.Hash) - require.Equal(t, "ping", exec.TaskKey) - require.Equal(t, execution.Status_Completed, exec.Status) + t.Run("get", func(t *testing.T) { + execGet, err := client.ExecutionClient.Get(ctx, &pb.GetExecutionRequest{Hash: executionHash}) + require.NoError(t, err) + require.True(t, execGet.Equal(execPingCompleted)) + }) - exec, err = client.ExecutionClient.Get(ctx, &pb.GetExecutionRequest{Hash: resp.Hash}) - require.NoError(t, err) - require.Equal(t, resp.Hash, exec.Hash) - require.Equal(t, "ping", exec.TaskKey) - require.Equal(t, execution.Status_Completed, exec.Status) + // t.Run("list", func(t *testing.T) { + // execList, err := client.ExecutionClient.List(ctx, &pb.GetExecutionRequest{Hash: executionHash}) + // require.NoError(t, err) + // require.True(t, execGet.Equal(execPingCompleted)) + // }) } diff --git a/e2e/testdata/e2e.config.yml b/e2e/testdata/e2e.config.yml index 86a3098ab..4191232af 100644 --- a/e2e/testdata/e2e.config.yml +++ b/e2e/testdata/e2e.config.yml @@ -1,5 +1,5 @@ log: - level: fatal + level: warn account: mnemonic: glimpse upon body vast economy give taxi yellow rabbit come click ranch chronic hammer sport near rotate charge lumber chicken cloud base thing forum diff --git a/e2e/testdata/test-service.json b/e2e/testdata/test-service.json index 21c25ec50..7845f637b 100644 --- a/e2e/testdata/test-service.json +++ b/e2e/testdata/test-service.json @@ -91,5 +91,5 @@ ] } ], - "source": "QmPkjHLWUwTVjJsy7ioFkxPL9yh7URYK2AUYYkTzJTmhJQ" + "source": "QmWyF2uxJ5d7VN98sDxYLq4HoyqcFv32wzCe9BzrJuHnCK" } diff --git a/e2e/testdata/test-service/go.mod b/e2e/testdata/test-service/go.mod index 96e3c2526..be8630d87 100644 --- a/e2e/testdata/test-service/go.mod +++ b/e2e/testdata/test-service/go.mod @@ -3,6 +3,6 @@ module main go 1.13 require ( - github.com/mesg-foundation/engine v0.15.0 + github.com/mesg-foundation/engine v0.15.1-0.20191119102502-3216db2fdeb9 google.golang.org/grpc v1.24.0 ) diff --git a/e2e/testdata/test-service/go.sum b/e2e/testdata/test-service/go.sum index 5ad77df53..c67b2de47 100644 --- a/e2e/testdata/test-service/go.sum +++ b/e2e/testdata/test-service/go.sum @@ -5,9 +5,10 @@ github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy86 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.20.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -29,6 +30,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -36,7 +38,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosmos/cosmos-sdk v0.37.0/go.mod h1:3b/k/Zd+YDuttSmEJdNkxga1H5EIiDUhSYeErAHQN7A= +github.com/cosmos/cosmos-sdk v0.37.3/go.mod h1:dAwYeOJ5ybRZg/OdRfiDy8q/cZq/GXQp9ZHAtz0E74I= github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/ledger-cosmos-go v0.10.3/go.mod h1:J8//BsAGTo3OC/vDLjMRFLW6q0WAaXvHnVc7ZmE8iUY= @@ -49,12 +51,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/cli v0.0.0-20191011045415-5d85cdacd257/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.0.0-20180803200506-eeea12db7a65/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20191006173954-0abbb9e4ebf1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -65,8 +71,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -74,6 +80,9 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -90,13 +99,16 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -111,7 +123,6 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -119,12 +130,13 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/lyft/protoc-gen-validate v0.1.0/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -132,14 +144,15 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mesg-foundation/engine v0.15.0 h1:EbZbYrDafGd0wkazQvRjM2qZ4BYRVH+Oun9OFLmol+s= -github.com/mesg-foundation/engine v0.15.0/go.mod h1:YMr8ecDOi7kxUKweWMUqBG4/5HzMQxBO8675ccNgAo8= +github.com/mesg-foundation/engine v0.15.1-0.20191119102502-3216db2fdeb9 h1:zXN9B47xm1gDoKokv1Ml7HG1bsLHUyHV5ouIQyPCsf8= +github.com/mesg-foundation/engine v0.15.1-0.20191119102502-3216db2fdeb9/go.mod h1:PKdZsf3GqJrgX5+E4/+DII3fVu/SrQDOaJmESl4jfWQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.1 h1:OJIdWOWYe2l5PQNgimGtuwHY8nDskvJ5vvs//YnzRLs= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -153,6 +166,7 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -160,7 +174,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= @@ -169,14 +182,10 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190227231451-bbced9601137/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= @@ -215,16 +224,18 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stumble/gorocksdb v0.0.3/go.mod h1:v6IHdFBXk5DJ1K4FZ0xi+eY737quiiBxYtSWXadLybY= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tendermint/btcd v0.1.1/go.mod h1:DC6/m53jtQzr/NFmMNEu0rxf18/ktVoVtMrnDD5pN+U= github.com/tendermint/crypto v0.0.0-20180820045704-3764759f34a5/go.mod h1:z4YtwM70uOnk8h0pjJYlj3zdYwi9l03By6iAIF5j/Pk= github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso= +github.com/tendermint/go-amino v0.15.0 h1:TC4e66P59W7ML9+bxio17CPKnxW3nKIRAYskntMAoRk= github.com/tendermint/go-amino v0.15.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/iavl v0.12.4/go.mod h1:8LHakzt8/0G3/I8FUU0ReNx98S/EP6eyPJkAUvEXT/o= github.com/tendermint/tendermint v0.32.1/go.mod h1:jmPDAKuNkev9793/ivn/fTBnfpA9mGBww8MPRNPNxnU= -github.com/tendermint/tendermint v0.32.2/go.mod h1:NwMyx58S8VJ7tEpFKqRVlVWKO9N9zjTHu+Dx96VsnOE= -github.com/tendermint/tendermint v0.32.3/go.mod h1:ZK2c29jl1QRYznIRyRWRDsmm1yvtPzBRT00x4t1JToY= +github.com/tendermint/tendermint v0.32.6/go.mod h1:D2+A3pNjY+Po72X0mTfaXorFhiVI8dh/Zg640FGyGtE= github.com/tendermint/tm-db v0.1.1/go.mod h1:0cPKWu2Mou3IlxecH+MEUSYc1Ch537alLe6CpFrKzgw= +github.com/tendermint/tm-db v0.2.0/go.mod h1:0cPKWu2Mou3IlxecH+MEUSYc1Ch537alLe6CpFrKzgw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -253,7 +264,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -303,28 +313,32 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 h1:iKtrH9Y8mcbADOP0YFaEMth7OfuHY9xHOwNj4znpM1A= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.30.0 h1:Wk0Z37oBmKj9/n+tPyBHZmeL19LaCoK3Qq48VwYENss= +gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/e2e/testdata/test-service/main.go b/e2e/testdata/test-service/main.go index 085aab5a7..4261bdc5e 100644 --- a/e2e/testdata/test-service/main.go +++ b/e2e/testdata/test-service/main.go @@ -87,7 +87,7 @@ func main() { stream, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ Filter: &pb.StreamExecutionRequest_Filter{ Statuses: []execution.Status{execution.Status_InProgress}, - InstanceHash: client.InstanceHash, + ExecutorHash: client.RunnerHash, }, }) if err != nil { diff --git a/go.mod b/go.mod index 288b9961e..a625d4a03 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.30.0 gopkg.in/yaml.v2 v2.2.4 - gotest.tools v2.2.0+incompatible // indirect + gotest.tools v2.2.0+incompatible ) go 1.13 From 1da0b08105849a78aacb02bcbb1d8f519bd6f676 Mon Sep 17 00:00:00 2001 From: krhubert Date: Tue, 19 Nov 2019 23:23:55 +0100 Subject: [PATCH 09/19] Refactor exeuction - remove/comment panics - handle channel closing - move filter related methods to api package - change app.handler return type - add valid method for hash - move some part to x packages --- core/main.go | 3 + cosmos/client.go | 60 +++----- cosmos/module.go | 25 ++-- cosmos/type.go | 9 +- hash/hash.go | 32 +++-- orchestrator/mocks/ExecutionSDK.go | 25 ++-- orchestrator/orchestrator.go | 8 +- orchestrator/type.go | 2 +- protobuf/api/execution.go | 63 +++++++++ .../api}/execution_test.go | 44 +++--- sdk/execution/backend.go | 6 +- sdk/execution/sdk.go | 129 ++++-------------- sdk/instance/backend.go | 2 +- sdk/ownership/backend.go | 2 +- sdk/runner/backend.go | 9 +- sdk/sdk.go | 2 +- sdk/service/backend.go | 4 +- server/grpc/api/execution.go | 18 +-- x/xrand/seed.go | 19 +++ x/xstrings/strings.go | 13 ++ 20 files changed, 240 insertions(+), 235 deletions(-) create mode 100644 protobuf/api/execution.go rename {sdk/execution => protobuf/api}/execution_test.go (52%) create mode 100644 x/xrand/seed.go diff --git a/core/main.go b/core/main.go index c8596ba66..10157b2bf 100644 --- a/core/main.go +++ b/core/main.go @@ -21,6 +21,7 @@ import ( "github.com/mesg-foundation/engine/version" "github.com/mesg-foundation/engine/x/xerrors" "github.com/mesg-foundation/engine/x/xnet" + "github.com/mesg-foundation/engine/x/xrand" "github.com/mesg-foundation/engine/x/xsignal" "github.com/sirupsen/logrus" tmtypes "github.com/tendermint/tendermint/types" @@ -114,6 +115,8 @@ func loadOrGenDevGenesis(app *cosmos.App, kb *cosmos.Keybase, cfg *config.Config } func main() { + xrand.SeedInit() + cfg, err := config.New() if err != nil { logrus.WithField("module", "main").Fatalln(err) diff --git a/cosmos/client.go b/cosmos/client.go index 8e0f65f9e..d53bebe94 100644 --- a/cosmos/client.go +++ b/cosmos/client.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "time" "github.com/cosmos/cosmos-sdk/crypto/keys" @@ -13,6 +12,7 @@ import ( "github.com/mesg-foundation/engine/codec" "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/x/xreflect" + "github.com/mesg-foundation/engine/x/xstrings" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/node" rpcclient "github.com/tendermint/tendermint/rpc/client" @@ -133,56 +133,32 @@ func (c *Client) BuildAndBroadcastMsg(msg sdktypes.Msg, accName, accPassword str } // Stream subscribes to the provided query and returns the hash of the matching ressources. -func (c *Client) Stream(query string) (<-chan hash.Hash, func() error, error) { - ctx, cancel := context.WithCancel(context.Background()) - // TODO: randStringRunes seems to complicated for this random string - subscriber := randStringRunes(8) // generates a new subscriber each time to be able to subscribe to the same query multiple time - eventStream, err := c.Subscribe(ctx, subscriber, query) +func (c *Client) Stream(query string) (chan hash.Hash, error) { + subscriber := xstrings.RandAsciiLetters(8) + eventStream, err := c.Subscribe(context.Background(), subscriber, query) if err != nil { - return nil, func() error { - cancel() - return nil - }, err - } - closer := func() error { - err := c.Unsubscribe(ctx, subscriber, query) - if err != nil { - return err - } - cancel() - return nil + return nil, err } - hashChan := make(chan hash.Hash) + + hashC := make(chan hash.Hash) go func() { - defer close(hashChan) for event := range eventStream { - ressHashes := event.Events[eventHashKey()] + ressHashes := event.Events[EventHashType] if len(ressHashes) != 1 { - // TODO: remove panic - panic(errors.New("there is no or more than one " + eventHashKey())) + // or panic(err) - grpc api do not support + // return the errors on the stream for now + // so besieds logging the error, it not + // much we can do here. same belove + continue } + hash, err := hash.Decode(ressHashes[0]) if err != nil { - // TODO: remove panic - panic(err) + continue } - hashChan <- hash + hashC <- hash } + c.Unsubscribe(context.Background(), subscriber, query) }() - return hashChan, closer, nil -} - -func init() { - rand.Seed(time.Now().UnixNano()) -} - -var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randStringRunes(n int) string { - b := make([]rune, n) - nbr := len(letterRunes) - for i := range b { - b[i] = letterRunes[rand.Intn(nbr)] - } - return string(b) + return hashC, nil } diff --git a/cosmos/module.go b/cosmos/module.go index 4dd660367..b2ba37e0c 100644 --- a/cosmos/module.go +++ b/cosmos/module.go @@ -33,7 +33,7 @@ type AppModule struct { } // Handler defines the core of the state transition function of an application. -type Handler func(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) +type Handler func(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) // Querier is responsible to answer to ABCI queries. type Querier func(request cosmostypes.Request, path []string, req abci.RequestQuery) (res interface{}, err error) @@ -107,17 +107,24 @@ func (m AppModule) NewHandler() cosmostypes.Handler { return func(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { hash, err := m.handler(request, msg) if err != nil { - if errsdk, ok := err.(cosmostypes.Error); ok { - return errsdk.Result() - } - return cosmostypes.ErrInternal(err.Error()).Result() + return err.Result() } + events := request.EventManager().Events() + events = events.AppendEvent( + cosmostypes.NewEvent( + cosmostypes.EventTypeMessage, + cosmostypes.NewAttribute(cosmostypes.AttributeKeyModule, m.name), + ), + ) + if hash != nil { - events = events.AppendEvents([]cosmostypes.Event{ - cosmostypes.NewEvent(cosmostypes.EventTypeMessage, cosmostypes.NewAttribute(cosmostypes.AttributeKeyModule, m.name)), - cosmostypes.NewEvent(cosmostypes.EventTypeMessage, cosmostypes.NewAttribute(eventHashAttr, hash.String())), - }) + events = events.AppendEvent( + cosmostypes.NewEvent( + cosmostypes.EventTypeMessage, + cosmostypes.NewAttribute(AttributeKeyHash, hash.String()), + ), + ) } return cosmostypes.Result{ Data: hash, diff --git a/cosmos/type.go b/cosmos/type.go index ec8b79f1c..cfdc09059 100644 --- a/cosmos/type.go +++ b/cosmos/type.go @@ -2,19 +2,18 @@ package cosmos import ( "fmt" - "strings" cosmostypes "github.com/cosmos/cosmos-sdk/types" sdktypes "github.com/cosmos/cosmos-sdk/types" ) +// common attribute keys. const ( - eventHashAttr = "hash" + AttributeKeyHash = "hash" ) -func eventHashKey() string { - return strings.Join([]string{cosmostypes.EventTypeMessage, eventHashAttr}, ".") -} +// EventHashType +var EventHashType = cosmostypes.EventTypeMessage + "." + AttributeKeyHash func EventActionQuery(msgType string) string { return fmt.Sprintf("%s.%s='%s'", sdktypes.EventTypeMessage, sdktypes.AttributeKeyAction, msgType) diff --git a/hash/hash.go b/hash/hash.go index 73349d89a..b5c2f5d1b 100644 --- a/hash/hash.go +++ b/hash/hash.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/binary" "encoding/json" + "errors" "fmt" "hash" @@ -19,12 +20,7 @@ var DefaultHash = sha256.New // size is a default size for hashing algorithm. var size = DefaultHash().Size() -func checkLen(data []byte) error { - if len(data) != size { - return fmt.Errorf("hash: invalid length") - } - return nil -} +var errInvalidLen = errors.New("hash: invalid length") // Digest represents the partial evaluation of a checksum. type Digest struct { @@ -87,8 +83,8 @@ func Decode(h string) (Hash, error) { if err != nil { return nil, fmt.Errorf("hash: %s", err) } - if err := checkLen(hash); err != nil { - return nil, err + if len(hash) != size { + return nil, errInvalidLen } return Hash(hash), nil } @@ -110,18 +106,24 @@ func (h Hash) Equal(h1 Hash) bool { // Marshal marshals hash into slice of bytes. It's used by protobuf. func (h Hash) Marshal() ([]byte, error) { - return h, checkLen(h) + if len(h) != size { + return nil, errInvalidLen + } + return h, nil } // MarshalTo marshals hash into slice of bytes. It's used by protobuf. func (h Hash) MarshalTo(data []byte) (int, error) { - return copy(data, h), checkLen(h) + if len(h) != size { + return 0, errInvalidLen + } + return copy(data, h), nil } // Unmarshal unmarshals slice of bytes into hash. It's used by protobuf. func (h *Hash) Unmarshal(data []byte) error { - if err := checkLen(data); err != nil { - return err + if len(data) != size { + return errInvalidLen } *h = make([]byte, len(data)) copy(*h, data) @@ -133,6 +135,12 @@ func (h Hash) Size() int { return len(h) } +// Valid checks if service hash length is valid. +// It treats empty hash as valid one. +func (h Hash) Valid() bool { + return len(h) == 0 || len(h) == size +} + // MarshalJSON mashals hash into encoded json string. func (h Hash) MarshalJSON() ([]byte, error) { return json.Marshal(base58.Encode(h)) diff --git a/orchestrator/mocks/ExecutionSDK.go b/orchestrator/mocks/ExecutionSDK.go index d7fd49bf8..cb360f645 100644 --- a/orchestrator/mocks/ExecutionSDK.go +++ b/orchestrator/mocks/ExecutionSDK.go @@ -59,33 +59,24 @@ func (_m *ExecutionSDK) Get(_a0 hash.Hash) (*execution.Execution, error) { } // Stream provides a mock function with given fields: req -func (_m *ExecutionSDK) Stream(req *api.StreamExecutionRequest) (<-chan *execution.Execution, func() error, error) { +func (_m *ExecutionSDK) Stream(req *api.StreamExecutionRequest) (chan *execution.Execution, error) { ret := _m.Called(req) - var r0 <-chan *execution.Execution - if rf, ok := ret.Get(0).(func(*api.StreamExecutionRequest) <-chan *execution.Execution); ok { + var r0 chan *execution.Execution + if rf, ok := ret.Get(0).(func(*api.StreamExecutionRequest) chan *execution.Execution); ok { r0 = rf(req) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan *execution.Execution) + r0 = ret.Get(0).(chan *execution.Execution) } } - var r1 func() error - if rf, ok := ret.Get(1).(func(*api.StreamExecutionRequest) func() error); ok { + var r1 error + if rf, ok := ret.Get(1).(func(*api.StreamExecutionRequest) error); ok { r1 = rf(req) } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(func() error) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(*api.StreamExecutionRequest) error); ok { - r2 = rf(req) - } else { - r2 = ret.Error(2) + r1 = ret.Error(1) } - return r0, r1, r2 + return r0, r1 } diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index ebff280ad..97d260b5b 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -33,15 +33,12 @@ func (s *Orchestrator) Start() error { return fmt.Errorf("process orchestrator already running") } s.eventStream = s.event.GetStream(nil) - executionStream, closer, err := s.execution.Stream(&api.StreamExecutionRequest{ + executionStream, err := s.execution.Stream(&api.StreamExecutionRequest{ Filter: &api.StreamExecutionRequest_Filter{ Statuses: []execution.Status{execution.Status_Completed}, }, }) if err != nil { - if err := closer(); err != nil { - panic(err) - } return err } s.executionStream = executionStream @@ -54,7 +51,8 @@ func (s *Orchestrator) Start() error { go s.execute(s.dependencyFilter(execution), execution, nil, execution.Outputs) } } - // TODO: manage closer + close(executionStream) + return nil } func (s *Orchestrator) eventFilter(event *event.Event) func(wf *process.Process, node *process.Process_Node) (bool, error) { diff --git a/orchestrator/type.go b/orchestrator/type.go index 7ead469c8..13341d0e4 100644 --- a/orchestrator/type.go +++ b/orchestrator/type.go @@ -12,7 +12,7 @@ import ( // ExecutionSDK execution interface needed for the orchestrator type ExecutionSDK interface { - Stream(req *api.StreamExecutionRequest) (<-chan *execution.Execution, func() error, error) + Stream(req *api.StreamExecutionRequest) (chan *execution.Execution, error) Get(hash hash.Hash) (*execution.Execution, error) Create(req *api.CreateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) } diff --git a/protobuf/api/execution.go b/protobuf/api/execution.go new file mode 100644 index 000000000..d4e321387 --- /dev/null +++ b/protobuf/api/execution.go @@ -0,0 +1,63 @@ +package api + +import ( + fmt "fmt" + + execution "github.com/mesg-foundation/engine/execution" + "github.com/mesg-foundation/engine/x/xstrings" +) + +// Validate checks if given filter is valid and returns error. +func (f *StreamExecutionRequest_Filter) Validate() error { + if f == nil { + return nil + } + + if !f.ExecutorHash.Valid() { + return fmt.Errorf("stream filter: executor hash is invalid") + } + + if !f.InstanceHash.Valid() { + return fmt.Errorf("stream filter: instance hash is invalid") + } + + // TODO: add validation (after adding in protobuf with print ascii) + // if f.TaskKey == "" || f.TaskKey == "*" || validation { + // return err + // } + + return nil +} + +// Match matches given execution with filter criteria. +func (f *StreamExecutionRequest_Filter) Match(e *execution.Execution) bool { + if f == nil { + return true + } + if !f.ExecutorHash.IsZero() && !f.ExecutorHash.Equal(e.ExecutorHash) { + return false + } + if !f.InstanceHash.IsZero() && !f.InstanceHash.Equal(e.InstanceHash) { + return false + } + if f.TaskKey != "" && f.TaskKey != "*" && f.TaskKey != e.TaskKey { + return false + } + + match := len(f.Statuses) == 0 + for _, status := range f.Statuses { + if status == e.Status { + match = true + break + } + } + if !match { + return false + } + for _, tag := range f.Tags { + if !xstrings.SliceContains(e.Tags, tag) { + return false + } + } + return true +} diff --git a/sdk/execution/execution_test.go b/protobuf/api/execution_test.go similarity index 52% rename from sdk/execution/execution_test.go rename to protobuf/api/execution_test.go index b96a3a1b9..81af9facf 100644 --- a/sdk/execution/execution_test.go +++ b/protobuf/api/execution_test.go @@ -1,17 +1,16 @@ -package executionsdk +package api import ( "testing" "github.com/mesg-foundation/engine/execution" "github.com/mesg-foundation/engine/hash" - "github.com/mesg-foundation/engine/protobuf/api" - "gotest.tools/assert" + "github.com/stretchr/testify/assert" ) func TestFilter(t *testing.T) { var tests = []struct { - f *api.StreamExecutionRequest_Filter + f *StreamExecutionRequest_Filter e *execution.Execution match bool }{ @@ -21,75 +20,75 @@ func TestFilter(t *testing.T) { true, }, { - &api.StreamExecutionRequest_Filter{}, + &StreamExecutionRequest_Filter{}, &execution.Execution{}, true, }, { - &api.StreamExecutionRequest_Filter{InstanceHash: hash.Int(1)}, + &StreamExecutionRequest_Filter{InstanceHash: hash.Int(1)}, &execution.Execution{InstanceHash: hash.Int(1)}, true, }, { - &api.StreamExecutionRequest_Filter{InstanceHash: hash.Int(1)}, + &StreamExecutionRequest_Filter{InstanceHash: hash.Int(1)}, &execution.Execution{InstanceHash: hash.Int(2)}, false, }, { - &api.StreamExecutionRequest_Filter{ExecutorHash: hash.Int(1)}, + &StreamExecutionRequest_Filter{ExecutorHash: hash.Int(1)}, &execution.Execution{ExecutorHash: hash.Int(1)}, true, }, { - &api.StreamExecutionRequest_Filter{ExecutorHash: hash.Int(1)}, + &StreamExecutionRequest_Filter{ExecutorHash: hash.Int(1)}, &execution.Execution{ExecutorHash: hash.Int(2)}, false, }, { - &api.StreamExecutionRequest_Filter{Statuses: []execution.Status{execution.Status_Created}}, + &StreamExecutionRequest_Filter{Statuses: []execution.Status{execution.Status_Created}}, &execution.Execution{Status: execution.Status_Created}, true, }, { - &api.StreamExecutionRequest_Filter{Statuses: []execution.Status{execution.Status_Created}}, + &StreamExecutionRequest_Filter{Statuses: []execution.Status{execution.Status_Created}}, &execution.Execution{Status: execution.Status_InProgress}, false, }, { - &api.StreamExecutionRequest_Filter{TaskKey: "0"}, + &StreamExecutionRequest_Filter{TaskKey: "0"}, &execution.Execution{TaskKey: "0"}, true, }, { - &api.StreamExecutionRequest_Filter{TaskKey: "*"}, + &StreamExecutionRequest_Filter{TaskKey: "*"}, &execution.Execution{TaskKey: "0"}, true, }, { - &api.StreamExecutionRequest_Filter{TaskKey: "0"}, + &StreamExecutionRequest_Filter{TaskKey: "0"}, &execution.Execution{TaskKey: "1"}, false, }, { - &api.StreamExecutionRequest_Filter{Tags: []string{"0"}}, + &StreamExecutionRequest_Filter{Tags: []string{"0"}}, &execution.Execution{Tags: []string{"0"}}, true, }, { - &api.StreamExecutionRequest_Filter{Tags: []string{"0", "1"}}, + &StreamExecutionRequest_Filter{Tags: []string{"0", "1"}}, &execution.Execution{Tags: []string{"0"}}, false, }, } for i, tt := range tests { - assert.Equal(t, tt.match, match(tt.f, tt.e), i) + assert.Equal(t, tt.match, tt.f.Match(tt.e), i) } } func TestValidateFilter(t *testing.T) { var tests = []struct { - f *api.StreamExecutionRequest_Filter + f *StreamExecutionRequest_Filter isError bool }{ { @@ -97,22 +96,21 @@ func TestValidateFilter(t *testing.T) { false, }, { - &api.StreamExecutionRequest_Filter{}, + &StreamExecutionRequest_Filter{}, false, }, { - &api.StreamExecutionRequest_Filter{ + &StreamExecutionRequest_Filter{ TaskKey: "not-exist", }, false, }, } - s := SDK{} for i, tt := range tests { if tt.isError { - assert.Error(t, s.validateFilter(tt.f), "", i) + assert.Error(t, tt.f.Validate(), "", i) } else { - assert.NilError(t, s.validateFilter(tt.f), i) + assert.NoError(t, tt.f.Validate(), i) } } } diff --git a/sdk/execution/backend.go b/sdk/execution/backend.go index 9ad681180..9f2eac6ef 100644 --- a/sdk/execution/backend.go +++ b/sdk/execution/backend.go @@ -48,18 +48,18 @@ func (s *Backend) SetProcessSDK(processSDK *processsdk.Process) { s.processSDK = processSDK } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { switch msg := msg.(type) { case msgCreateExecution: exec, err := s.Create(request, msg) if err != nil { - return nil, err + return nil, cosmostypes.ErrInternal(err.Error()) } return exec.Hash, nil case msgUpdateExecution: exec, err := s.Update(request, msg) if err != nil { - return nil, err + return nil, cosmostypes.ErrInternal(err.Error()) } return exec.Hash, nil default: diff --git a/sdk/execution/sdk.go b/sdk/execution/sdk.go index d9bac8891..4203ed921 100644 --- a/sdk/execution/sdk.go +++ b/sdk/execution/sdk.go @@ -3,35 +3,31 @@ package executionsdk import ( "fmt" - cosmostypes "github.com/cosmos/cosmos-sdk/types" "github.com/mesg-foundation/engine/cosmos" "github.com/mesg-foundation/engine/execution" "github.com/mesg-foundation/engine/hash" - "github.com/mesg-foundation/engine/instance" "github.com/mesg-foundation/engine/protobuf/api" - "github.com/mesg-foundation/engine/runner" - accountsdk "github.com/mesg-foundation/engine/sdk/account" instancesdk "github.com/mesg-foundation/engine/sdk/instance" runnersdk "github.com/mesg-foundation/engine/sdk/runner" servicesdk "github.com/mesg-foundation/engine/sdk/service" - "github.com/mesg-foundation/engine/x/xstrings" "github.com/tendermint/tendermint/mempool" ) // SDK is the execution sdk. type SDK struct { - client *cosmos.Client - accountSDK *accountsdk.SDK + client *cosmos.Client + kb *cosmos.Keybase + serviceSDK *servicesdk.SDK instanceSDK *instancesdk.SDK runnerSDK *runnersdk.SDK } // New returns the execution sdk. -func New(client *cosmos.Client, accountSDK *accountsdk.SDK, serviceSDK *servicesdk.SDK, instanceSDK *instancesdk.SDK, runnerSDK *runnersdk.SDK) *SDK { +func New(client *cosmos.Client, kb *cosmos.Keybase, serviceSDK *servicesdk.SDK, instanceSDK *instancesdk.SDK, runnerSDK *runnersdk.SDK) *SDK { sdk := &SDK{ client: client, - accountSDK: accountSDK, + kb: kb, serviceSDK: serviceSDK, instanceSDK: instanceSDK, runnerSDK: runnerSDK, @@ -41,15 +37,12 @@ func New(client *cosmos.Client, accountSDK *accountsdk.SDK, serviceSDK *services // Create creates a new execution. func (s *SDK) Create(req *api.CreateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) { - account, err := s.accountSDK.Get(accountName) + acc, err := s.kb.Get(accountName) if err != nil { return nil, err } - signer, err := cosmostypes.AccAddressFromBech32(account.Address) - if err != nil { - return nil, err - } - msg := newMsgCreateExecution(req, signer) + + msg := newMsgCreateExecution(req, acc.GetAddress()) tx, err := s.client.BuildAndBroadcastMsg(msg, accountName, accountPassword) if err != nil { if err == mempool.ErrTxInCache { @@ -62,15 +55,11 @@ func (s *SDK) Create(req *api.CreateExecutionRequest, accountName, accountPasswo // Update updates a execution. func (s *SDK) Update(req *api.UpdateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) { - account, err := s.accountSDK.Get(accountName) + acc, err := s.kb.Get(accountName) if err != nil { return nil, err } - executor, err := cosmostypes.AccAddressFromBech32(account.Address) - if err != nil { - return nil, err - } - msg := newMsgUpdateExecution(req, executor) + msg := newMsgUpdateExecution(req, acc.GetAddress()) tx, err := s.client.BuildAndBroadcastMsg(msg, accountName, accountPassword) if err != nil { if err == mempool.ErrTxInCache { @@ -100,96 +89,32 @@ func (s *SDK) List() ([]*execution.Execution, error) { } // Stream returns execution that matches given hash. -func (s *SDK) Stream(req *api.StreamExecutionRequest) (<-chan *execution.Execution, func() error, error) { - if err := s.validateFilter(req.Filter); err != nil { - return nil, func() error { return nil }, err +func (s *SDK) Stream(req *api.StreamExecutionRequest) (chan *execution.Execution, error) { + if err := req.Filter.Validate(); err != nil { + return nil, err } - stream, closer, err := s.client.Stream(cosmos.EventModuleQuery(backendName)) + + stream, err := s.client.Stream(cosmos.EventModuleQuery(backendName)) if err != nil { - return nil, closer, err + return nil, err } - execChan := make(chan *execution.Execution) + + execC := make(chan *execution.Execution) go func() { - defer close(execChan) for hash := range stream { exec, err := s.Get(hash) if err != nil { - // TODO: remove panic - panic(err) + // or panic(err) - grpc api do not support + // return the errors on the stream for now + // so besieds logging the error, it not + // much we can do here. + continue } - if match(req.Filter, exec) { - execChan <- exec + if req.Filter.Match(exec) { + execC <- exec } } + close(execC) }() - return execChan, closer, nil -} - -func (s *SDK) validateFilter(f *api.StreamExecutionRequest_Filter) error { - if f == nil { - return nil - } - var err error - var run *runner.Runner - if !f.ExecutorHash.IsZero() { - if run, err = s.runnerSDK.Get(f.ExecutorHash); err != nil { - return err - } - } - var inst *instance.Instance - if !f.InstanceHash.IsZero() { - if inst, err = s.instanceSDK.Get(f.InstanceHash); err != nil { - return err - } - } - if (f.TaskKey == "" || f.TaskKey == "*") || (inst == nil && run == nil) { - return nil - } - // check task key if at least instance or runner is set - if inst == nil && run != nil { - inst, err = s.instanceSDK.Get(run.InstanceHash) - if err != nil { - return err - } - } - srv, err := s.serviceSDK.Get(inst.ServiceHash) - if err != nil { - return err - } - if _, err := srv.GetTask(f.TaskKey); err != nil { - return err - } - return nil -} - -// Match matches an execution against a filter. -func match(f *api.StreamExecutionRequest_Filter, e *execution.Execution) bool { - if f == nil { - return true - } - if !f.ExecutorHash.IsZero() && !f.ExecutorHash.Equal(e.ExecutorHash) { - return false - } - if !f.InstanceHash.IsZero() && !f.InstanceHash.Equal(e.InstanceHash) { - return false - } - if f.TaskKey != "" && f.TaskKey != "*" && f.TaskKey != e.TaskKey { - return false - } - match := len(f.Statuses) == 0 - for _, status := range f.Statuses { - if status == e.Status { - match = true - break - } - } - if !match { - return false - } - for _, tag := range f.Tags { - if !xstrings.SliceContains(e.Tags, tag) { - return false - } - } - return true + return execC, nil } diff --git a/sdk/instance/backend.go b/sdk/instance/backend.go index 69023fe11..00eca05e9 100644 --- a/sdk/instance/backend.go +++ b/sdk/instance/backend.go @@ -32,7 +32,7 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { errmsg := fmt.Sprintf("Unrecognized instance Msg type: %v", msg.Type()) return nil, cosmostypes.ErrUnknownRequest(errmsg) } diff --git a/sdk/ownership/backend.go b/sdk/ownership/backend.go index a0fc01a25..8874122db 100644 --- a/sdk/ownership/backend.go +++ b/sdk/ownership/backend.go @@ -31,7 +31,7 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { errmsg := fmt.Sprintf("Unrecognized ownership Msg type: %v", msg.Type()) return nil, cosmostypes.ErrUnknownRequest(errmsg) } diff --git a/sdk/runner/backend.go b/sdk/runner/backend.go index 5baff0474..78edd44a1 100644 --- a/sdk/runner/backend.go +++ b/sdk/runner/backend.go @@ -35,16 +35,19 @@ func NewBackend(appFactory *cosmos.AppFactory, instanceBack *instancesdk.Backend return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { switch msg := msg.(type) { case msgCreateRunner: run, err := s.Create(request, &msg) if err != nil { - return nil, err + return run.Hash, cosmostypes.ErrInternal(err.Error()) } return run.Hash, nil case msgDeleteRunner: - return nil, s.Delete(request, &msg) + if err := s.Delete(request, &msg); err != nil { + return nil, cosmostypes.ErrInternal(err.Error()) + } + return nil, nil default: errmsg := fmt.Sprintf("Unrecognized runner Msg type: %v", msg.Type()) return nil, cosmostypes.ErrUnknownRequest(errmsg) diff --git a/sdk/sdk.go b/sdk/sdk.go index 57a7aac86..f8fb9edf2 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -36,7 +36,7 @@ func New(client *cosmos.Client, kb *cosmos.Keybase, processDB database.ProcessDB instanceSDK := instancesdk.New(client) runnerSDK := runnersdk.New(client, accountSDK, serviceSDK, instanceSDK, container, engineName, port, ipfsEndpoint) processSDK := processesdk.New(instanceSDK, processDB) - executionSDK := executionsdk.New(client, accountSDK, serviceSDK, instanceSDK, runnerSDK) + executionSDK := executionsdk.New(client, kb, serviceSDK, instanceSDK, runnerSDK) eventSDK := eventsdk.New(ps, serviceSDK, instanceSDK) return &SDK{ Service: serviceSDK, diff --git a/sdk/service/backend.go b/sdk/service/backend.go index b2affff66..3060f09d1 100644 --- a/sdk/service/backend.go +++ b/sdk/service/backend.go @@ -37,12 +37,12 @@ func NewBackend(appFactory *cosmos.AppFactory, ownerships *ownershipsdk.Backend) return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { switch msg := msg.(type) { case msgCreateService: srv, err := s.Create(request, &msg) if err != nil { - return nil, err + return nil, cosmostypes.ErrInternal(err.Error()) } return srv.Hash, nil default: diff --git a/server/grpc/api/execution.go b/server/grpc/api/execution.go index 42f324230..440e55df5 100644 --- a/server/grpc/api/execution.go +++ b/server/grpc/api/execution.go @@ -42,20 +42,22 @@ func (s *ExecutionServer) Get(ctx context.Context, req *api.GetExecutionRequest) // Stream returns stream of executions. func (s *ExecutionServer) Stream(req *api.StreamExecutionRequest, resp api.Execution_StreamServer) error { - stream, closer, err := s.sdk.Execution.Stream(req) - defer func() { - err := closer() - if err != nil { - // TODO: remove panic - panic(err) - } - }() + stream, err := s.sdk.Execution.Stream(req) if err != nil { return err } + defer close(stream) + if err := acknowledgement.SetStreamReady(resp); err != nil { return err } + + // TODO: + // There is possible deadlock. If the client close the connection, + // but there will be no messages in the stream, then this for will + // wait and consume resources forever. Some ACK mechnizm needs to be + // implemented on server/client side to get notify if the conneciton + // wasn't closed. for exec := range stream { if err := resp.Send(exec); err != nil { return err diff --git a/x/xrand/seed.go b/x/xrand/seed.go new file mode 100644 index 000000000..9fee766e3 --- /dev/null +++ b/x/xrand/seed.go @@ -0,0 +1,19 @@ +package xrand + +import ( + crypto_rand "crypto/rand" + "encoding/binary" + "math/rand" + "time" +) + +// SeedInit initialize rand package with secure random number. +func SeedInit() { + var b [8]byte + if _, err := crypto_rand.Read(b[:]); err == nil { + rand.Seed(int64(binary.LittleEndian.Uint64(b[:]))) + } else { + // fallback to unix nano time + rand.Seed(time.Now().UTC().UnixNano()) + } +} diff --git a/x/xstrings/strings.go b/x/xstrings/strings.go index 8899475a1..854af7ccd 100644 --- a/x/xstrings/strings.go +++ b/x/xstrings/strings.go @@ -1,5 +1,7 @@ package xstrings +import "math/rand" + // SliceContains returns true if slice a contains e element, false otherwise. func SliceContains(a []string, e string) bool { for _, s := range a { @@ -30,3 +32,14 @@ func SliceIndex(a []string, e string) int { } return -1 } + +var asciiletters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// RandAsciiLetters generates random string from ascii letters. +func RandAsciiLetters(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = asciiletters[rand.Intn(len(asciiletters))] + } + return string(b) +} From 158e204a106aed0d3ccfa5dc50bdce944cdd7859 Mon Sep 17 00:00:00 2001 From: Hubert Krauze Date: Wed, 20 Nov 2019 09:30:34 +0100 Subject: [PATCH 10/19] Select context done in execution stream --- server/grpc/api/execution.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/server/grpc/api/execution.go b/server/grpc/api/execution.go index 440e55df5..f3d38fcd1 100644 --- a/server/grpc/api/execution.go +++ b/server/grpc/api/execution.go @@ -52,18 +52,16 @@ func (s *ExecutionServer) Stream(req *api.StreamExecutionRequest, resp api.Execu return err } - // TODO: - // There is possible deadlock. If the client close the connection, - // but there will be no messages in the stream, then this for will - // wait and consume resources forever. Some ACK mechnizm needs to be - // implemented on server/client side to get notify if the conneciton - // wasn't closed. - for exec := range stream { - if err := resp.Send(exec); err != nil { - return err + for { + select { + case exec := <-stream: + if err := resp.Send(exec); err != nil { + return err + } + case <-resp.Context().Done(): + return resp.Context().Err() } } - return nil } // Update updates execution from given hash. From 237e85d72440d7f5dddb36fa99f7766ef1c7d7bf Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Wed, 20 Nov 2019 18:12:15 +0700 Subject: [PATCH 11/19] remove instanceHash from CreateExecutionRequest --- e2e/execution_test.go | 1 - orchestrator/orchestrator.go | 1 - protobuf/api/execution.pb.go | 84 ++++++++++++++++++------------------ protobuf/api/execution.proto | 4 -- sdk/backend.go | 2 +- sdk/execution/backend.go | 13 ++++-- 6 files changed, 52 insertions(+), 53 deletions(-) diff --git a/e2e/execution_test.go b/e2e/execution_test.go index f524a0d5b..e7e4d8984 100644 --- a/e2e/execution_test.go +++ b/e2e/execution_test.go @@ -59,7 +59,6 @@ func testExecution(t *testing.T) { require.NoError(t, err) acknowledgement.WaitForStreamToBeReady(stream) resp, err := client.ExecutionClient.Create(ctx, &pb.CreateExecutionRequest{ - InstanceHash: testInstanceHash, TaskKey: "ping", EventHash: hash.Int(1), ExecutorHash: testRunnerHash, diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index 97d260b5b..384f3e22b 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -208,7 +208,6 @@ func (s *Orchestrator) processTask(task *process.Process_Node_Task, wf *process. executor := executors[rand.Intn(len(executors))] _, err = s.execution.Create(&api.CreateExecutionRequest{ ProcessHash: wf.Hash, - InstanceHash: task.InstanceHash, EventHash: eventHash, ParentHash: execHash, StepID: task.Key, diff --git a/protobuf/api/execution.pb.go b/protobuf/api/execution.pb.go index 47b196efc..14ff65587 100644 --- a/protobuf/api/execution.pb.go +++ b/protobuf/api/execution.pb.go @@ -30,7 +30,6 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package // CreateExecutionRequest defines request to create a single execution. type CreateExecutionRequest struct { - InstanceHash github_com_mesg_foundation_engine_hash.Hash `protobuf:"bytes,1,opt,name=instanceHash,proto3,customtype=github.com/mesg-foundation/engine/hash.Hash" json:"instanceHash"` TaskKey string `protobuf:"bytes,2,opt,name=taskKey,proto3" json:"taskKey,omitempty"` Inputs *types.Struct `protobuf:"bytes,3,opt,name=inputs,proto3" json:"inputs,omitempty"` Tags []string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty"` @@ -463,48 +462,47 @@ func init() { func init() { proto.RegisterFile("protobuf/api/execution.proto", fileDescriptor_96e2c86581f82f05) } var fileDescriptor_96e2c86581f82f05 = []byte{ - // 641 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0xcf, 0x6e, 0xd3, 0x4e, - 0x10, 0x8e, 0xf3, 0xc7, 0x4d, 0xa6, 0xd5, 0xef, 0xb0, 0x3f, 0x9a, 0x9a, 0x00, 0xad, 0xf1, 0x85, - 0x48, 0xa8, 0x36, 0xa4, 0xe2, 0x88, 0x90, 0x0a, 0xa5, 0xad, 0x40, 0x20, 0x1c, 0x55, 0x48, 0x1c, - 0x90, 0x36, 0xe9, 0xd4, 0xb1, 0x68, 0xbd, 0x66, 0x77, 0x8d, 0xe8, 0x9b, 0xf0, 0x24, 0x1c, 0x39, - 0xf3, 0x0c, 0x1c, 0xfa, 0x14, 0x9c, 0x11, 0xf2, 0xae, 0xed, 0xb8, 0xb1, 0xdb, 0x43, 0x5b, 0x6e, - 0x1e, 0xcf, 0xcc, 0xb7, 0xf3, 0xe7, 0xfb, 0x06, 0xee, 0xc6, 0x9c, 0x49, 0x36, 0x49, 0x8e, 0x3c, - 0x1a, 0x87, 0x1e, 0x7e, 0xc5, 0x69, 0x22, 0x43, 0x16, 0xb9, 0xea, 0x37, 0xe9, 0x9e, 0xa0, 0x08, - 0x5c, 0x1a, 0x87, 0x03, 0x27, 0x60, 0x01, 0xf3, 0x8a, 0xe0, 0xd4, 0x52, 0x86, 0xfa, 0xd2, 0xd1, - 0x83, 0x3b, 0x85, 0x5b, 0x9e, 0xc6, 0x28, 0x3c, 0x21, 0x79, 0x32, 0x95, 0x99, 0x73, 0x7d, 0xc1, - 0xb9, 0xf0, 0x94, 0xf3, 0xa3, 0x0d, 0xfd, 0xe7, 0x1c, 0xa9, 0xc4, 0x9d, 0xdc, 0xe3, 0xe3, 0xe7, - 0x04, 0x85, 0x24, 0xef, 0x61, 0x25, 0x8c, 0x84, 0xa4, 0xd1, 0x14, 0xf7, 0xa8, 0x98, 0x59, 0x86, - 0x6d, 0x0c, 0x57, 0xb6, 0xb7, 0x7e, 0x9e, 0x6d, 0x34, 0x7e, 0x9d, 0x6d, 0x3c, 0x0c, 0x42, 0x39, - 0x4b, 0x26, 0xee, 0x94, 0x9d, 0x78, 0x69, 0xb9, 0x9b, 0x47, 0x2c, 0x89, 0x0e, 0x69, 0x0a, 0xe0, - 0x61, 0x14, 0x84, 0x11, 0x7a, 0x33, 0x2a, 0x66, 0x6e, 0x9a, 0xea, 0x9f, 0x03, 0x22, 0x16, 0x2c, - 0x49, 0x2a, 0x3e, 0xbd, 0xc2, 0x53, 0xab, 0x69, 0x1b, 0xc3, 0x9e, 0x9f, 0x9b, 0x64, 0x13, 0xcc, - 0x30, 0x8a, 0x13, 0x29, 0xac, 0x96, 0x6d, 0x0c, 0x97, 0x47, 0xab, 0xae, 0x9a, 0x44, 0xde, 0x83, - 0x3b, 0x56, 0xad, 0xf9, 0x59, 0x10, 0x21, 0xd0, 0x96, 0x34, 0x10, 0x56, 0xdb, 0x6e, 0x0d, 0x7b, - 0xbe, 0xfa, 0x26, 0x63, 0x80, 0x98, 0x72, 0x8c, 0xa4, 0xaa, 0xb9, 0x73, 0xf5, 0x9a, 0x4b, 0x30, - 0xe4, 0x1d, 0xf4, 0xf0, 0x4b, 0x8e, 0x69, 0x5e, 0x1d, 0x73, 0x8e, 0x42, 0x0e, 0x60, 0x39, 0xe6, - 0x6c, 0x8a, 0x42, 0x28, 0xd0, 0xa5, 0xab, 0x83, 0x96, 0x71, 0x48, 0x1f, 0x4c, 0x21, 0x31, 0xde, - 0x7f, 0x61, 0x75, 0xd5, 0x68, 0x33, 0x2b, 0x5d, 0xa6, 0x5e, 0x3d, 0xe3, 0xea, 0xbd, 0xde, 0x35, - 0x96, 0x59, 0x06, 0x72, 0x26, 0xb0, 0x56, 0xe1, 0x8f, 0x88, 0x59, 0x24, 0x90, 0xec, 0x42, 0x7b, - 0x76, 0x4d, 0xe2, 0x28, 0x00, 0xe7, 0x23, 0xfc, 0xbf, 0x8b, 0xb2, 0x42, 0xd0, 0x1b, 0xc3, 0xff, - 0xdd, 0x84, 0xfe, 0x58, 0x72, 0xa4, 0x27, 0x95, 0x37, 0x9e, 0x81, 0x79, 0x14, 0x1e, 0x4b, 0xe4, - 0xea, 0x95, 0xe5, 0xd1, 0x03, 0x37, 0xd7, 0xa6, 0x5b, 0x9f, 0xe1, 0xbe, 0x54, 0xe1, 0x7e, 0x96, - 0x36, 0xf8, 0xd6, 0x04, 0x53, 0xff, 0x22, 0x2e, 0x74, 0x85, 0xa4, 0x32, 0x11, 0x28, 0x2c, 0xc3, - 0x6e, 0x0d, 0xff, 0x1b, 0x11, 0x8d, 0xa6, 0xa4, 0xe9, 0x8e, 0x95, 0xcf, 0x2f, 0x62, 0x2a, 0x02, - 0x6c, 0xfe, 0x03, 0x01, 0xb6, 0xce, 0x0b, 0xb0, 0x4e, 0x51, 0x8b, 0xd4, 0xe9, 0xdc, 0x14, 0x75, - 0xbe, 0x1b, 0xd0, 0x3f, 0x88, 0x0f, 0xeb, 0x6e, 0xcf, 0x4d, 0xad, 0x96, 0x3c, 0x86, 0x25, 0x96, - 0x48, 0x75, 0x52, 0x9a, 0x97, 0x9c, 0x94, 0xbd, 0x86, 0x9f, 0xc7, 0x91, 0x3e, 0x74, 0x90, 0x73, - 0xc6, 0xf5, 0x6c, 0xf6, 0x1a, 0xbe, 0x36, 0xb7, 0xbb, 0x60, 0x72, 0x14, 0xc9, 0xb1, 0x74, 0x6e, - 0xc3, 0x5a, 0xa5, 0x6e, 0xcd, 0x79, 0xa7, 0x0f, 0xb7, 0x5e, 0x87, 0xa2, 0xc2, 0x55, 0xe7, 0x0d, - 0xac, 0x2e, 0xfc, 0xcf, 0x44, 0xf2, 0x04, 0xa0, 0xb8, 0xc9, 0x9a, 0x16, 0x45, 0x8d, 0x9a, 0x16, - 0xf3, 0x94, 0x52, 0xe0, 0xe8, 0x4f, 0x13, 0x7a, 0x85, 0x87, 0xbc, 0x05, 0x53, 0x8b, 0x90, 0xd8, - 0x73, 0x7e, 0xd6, 0x9f, 0xf5, 0xc1, 0xfd, 0x4b, 0x22, 0xb2, 0x26, 0x1a, 0xe4, 0x29, 0xb4, 0x76, - 0x51, 0x92, 0x7b, 0xf3, 0xd8, 0x1a, 0x01, 0x0e, 0xea, 0xeb, 0x74, 0x1a, 0x64, 0x1f, 0xda, 0x69, - 0xb7, 0x64, 0x7d, 0x9e, 0x5f, 0x37, 0x95, 0xc1, 0xc6, 0x85, 0xfe, 0xa2, 0x92, 0x1d, 0x30, 0xb5, - 0xd0, 0xca, 0xad, 0xd5, 0x4b, 0xef, 0xc2, 0x7a, 0x1e, 0x19, 0xe9, 0x84, 0xf4, 0xca, 0xca, 0x30, - 0xf5, 0xe4, 0x2b, 0x4f, 0xe8, 0xa2, 0x35, 0x37, 0xb6, 0x3b, 0x1f, 0x5a, 0x34, 0x0e, 0x27, 0xa6, - 0x22, 0xd2, 0xd6, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x08, 0x34, 0x0c, 0x6c, 0xd1, 0x07, 0x00, - 0x00, + // 638 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xc1, 0x6e, 0xd3, 0x4c, + 0x10, 0x8e, 0xe3, 0xc4, 0x4d, 0xa6, 0xd5, 0x7f, 0xd8, 0x9f, 0xa6, 0x26, 0x40, 0x6b, 0x7c, 0x21, + 0x12, 0xaa, 0x0d, 0xa9, 0x38, 0x22, 0xa4, 0x42, 0x69, 0x2b, 0x10, 0x08, 0x47, 0x15, 0x12, 0x07, + 0xa4, 0x4d, 0x3a, 0x75, 0x2c, 0x5a, 0xaf, 0xd9, 0x5d, 0x23, 0xfa, 0x26, 0x3c, 0x09, 0xcf, 0xc0, + 0x33, 0x70, 0xe8, 0x8d, 0x37, 0xe0, 0x8c, 0x90, 0x77, 0x6d, 0xc7, 0x4d, 0xdc, 0x1e, 0x42, 0x6f, + 0x1e, 0xcf, 0xcc, 0x37, 0xb3, 0x33, 0xdf, 0x37, 0x70, 0x37, 0xe1, 0x4c, 0xb2, 0x71, 0x7a, 0xe2, + 0xd3, 0x24, 0xf2, 0xf1, 0x2b, 0x4e, 0x52, 0x19, 0xb1, 0xd8, 0x53, 0xbf, 0x49, 0xe7, 0x0c, 0x45, + 0xe8, 0xd1, 0x24, 0xea, 0xbb, 0x21, 0x0b, 0x99, 0x5f, 0x06, 0x67, 0x96, 0x32, 0xd4, 0x97, 0x8e, + 0xee, 0xdf, 0x29, 0xdd, 0xf2, 0x3c, 0x41, 0xe1, 0x0b, 0xc9, 0xd3, 0x89, 0xcc, 0x9d, 0x9b, 0x73, + 0xce, 0xb9, 0x52, 0xee, 0x2f, 0x13, 0x7a, 0xcf, 0x39, 0x52, 0x89, 0x7b, 0x85, 0x27, 0xc0, 0xcf, + 0x29, 0x0a, 0x49, 0x6c, 0x58, 0x91, 0x54, 0x7c, 0x7a, 0x85, 0xe7, 0x76, 0xd3, 0x31, 0x06, 0xdd, + 0xa0, 0x30, 0xc9, 0x36, 0x58, 0x51, 0x9c, 0xa4, 0x52, 0xd8, 0xa6, 0x63, 0x0c, 0x56, 0x87, 0xeb, + 0x9e, 0x6a, 0xb8, 0x28, 0xe5, 0x8d, 0x54, 0x07, 0x41, 0x1e, 0x44, 0x08, 0xb4, 0x24, 0x0d, 0x85, + 0xdd, 0x72, 0xcc, 0x41, 0x37, 0x50, 0xdf, 0x64, 0x04, 0x90, 0x50, 0x8e, 0xb1, 0x3c, 0xa0, 0x62, + 0x6a, 0xb7, 0x1d, 0x63, 0xb0, 0xb6, 0xbb, 0xf3, 0xe3, 0x62, 0xab, 0xf1, 0xf3, 0x62, 0xeb, 0x61, + 0x18, 0xc9, 0x69, 0x3a, 0xf6, 0x26, 0xec, 0xcc, 0xcf, 0x80, 0xb7, 0x4f, 0x58, 0x1a, 0x1f, 0xd3, + 0xac, 0x37, 0x1f, 0xe3, 0x30, 0x8a, 0xd1, 0x9f, 0x52, 0x31, 0xf5, 0xb2, 0xd4, 0xa0, 0x02, 0x43, + 0xde, 0x41, 0x17, 0xbf, 0x14, 0x98, 0xd6, 0xf2, 0x98, 0x33, 0x14, 0x72, 0x04, 0xab, 0x09, 0x67, + 0x13, 0x14, 0x42, 0x81, 0xae, 0x2c, 0x0f, 0x5a, 0xc5, 0x21, 0x3d, 0xb0, 0x84, 0xc4, 0xe4, 0xf0, + 0x85, 0xdd, 0x51, 0xa3, 0xcd, 0x2d, 0xf2, 0x1e, 0xd6, 0xf4, 0x86, 0x18, 0x57, 0xf5, 0xba, 0xcb, + 0xd7, 0xbb, 0x04, 0xe4, 0x8e, 0x61, 0x63, 0x61, 0xcd, 0x22, 0x61, 0xb1, 0x40, 0xb2, 0x0f, 0xad, + 0x2c, 0xcb, 0x36, 0x96, 0xaf, 0xa5, 0x00, 0xdc, 0x8f, 0xf0, 0xff, 0x3e, 0xca, 0x05, 0x1e, 0xdd, + 0x18, 0xfe, 0xef, 0x26, 0xf4, 0x46, 0x92, 0x23, 0x3d, 0x5b, 0xa8, 0xf1, 0x0c, 0xac, 0x93, 0xe8, + 0x54, 0x22, 0x57, 0x55, 0x56, 0x87, 0x0f, 0xbc, 0x42, 0x42, 0x5e, 0x7d, 0x86, 0xf7, 0x52, 0x85, + 0x07, 0x79, 0x5a, 0xff, 0x5b, 0x13, 0x2c, 0xfd, 0x8b, 0x78, 0xd0, 0x11, 0x92, 0xca, 0x54, 0xa0, + 0xb0, 0x0d, 0xc7, 0x1c, 0xfc, 0x37, 0x24, 0x1a, 0x4d, 0x29, 0xc8, 0x1b, 0x29, 0x5f, 0x50, 0xc6, + 0x64, 0x3b, 0x8b, 0x62, 0x21, 0x69, 0x3c, 0x41, 0xb5, 0xb3, 0xe6, 0x3f, 0xec, 0xac, 0x0a, 0x54, + 0x15, 0xa0, 0x79, 0x59, 0x80, 0x75, 0x8a, 0x9a, 0xa7, 0x4e, 0xfb, 0xa6, 0xa8, 0xf3, 0xdd, 0x80, + 0xde, 0x51, 0x72, 0x5c, 0x77, 0x22, 0x6e, 0x6a, 0xb5, 0xe4, 0x31, 0xac, 0xb0, 0x54, 0xaa, 0x93, + 0xd2, 0xbc, 0xe6, 0xa4, 0x1c, 0x34, 0x82, 0x22, 0x8e, 0xf4, 0xa0, 0x8d, 0x9c, 0x33, 0xae, 0x67, + 0x73, 0xd0, 0x08, 0xb4, 0xb9, 0xdb, 0x01, 0x8b, 0xa3, 0x48, 0x4f, 0xa5, 0x7b, 0x1b, 0x36, 0x16, + 0xfa, 0xd6, 0x9c, 0x77, 0x7b, 0x70, 0xeb, 0x75, 0x24, 0x16, 0xb8, 0xea, 0xbe, 0x81, 0xf5, 0xb9, + 0xff, 0xb9, 0x48, 0x9e, 0x00, 0x94, 0xa7, 0x53, 0xd3, 0xa2, 0xec, 0x51, 0xd3, 0x62, 0x96, 0x52, + 0x09, 0x1c, 0xfe, 0x69, 0x42, 0xb7, 0xf4, 0x90, 0xb7, 0x60, 0x69, 0x11, 0x12, 0x67, 0xc6, 0xcf, + 0xfa, 0xeb, 0xdb, 0xbf, 0x7f, 0x4d, 0x44, 0xfe, 0x88, 0x06, 0x79, 0x0a, 0xe6, 0x3e, 0x4a, 0x72, + 0x6f, 0x16, 0x5b, 0x23, 0xc0, 0x7e, 0x7d, 0x9f, 0x6e, 0x83, 0x1c, 0x42, 0x2b, 0x7b, 0x2d, 0xd9, + 0x9c, 0xe5, 0xd7, 0x4d, 0xa5, 0xbf, 0x75, 0xa5, 0xbf, 0xec, 0x64, 0x0f, 0x2c, 0x2d, 0xb4, 0xea, + 0xd3, 0xea, 0xa5, 0x77, 0x65, 0x3f, 0x8f, 0x8c, 0x6c, 0x42, 0x7a, 0x65, 0x55, 0x98, 0x7a, 0xf2, + 0x55, 0x27, 0x74, 0xd5, 0x9a, 0x1b, 0xbb, 0xed, 0x0f, 0x26, 0x4d, 0xa2, 0xb1, 0xa5, 0x88, 0xb4, + 0xf3, 0x37, 0x00, 0x00, 0xff, 0xff, 0x44, 0x6c, 0xe8, 0x6c, 0x78, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/protobuf/api/execution.proto b/protobuf/api/execution.proto index b9d9dee73..98df6fd40 100644 --- a/protobuf/api/execution.proto +++ b/protobuf/api/execution.proto @@ -33,10 +33,6 @@ service Execution { // CreateExecutionRequest defines request to create a single execution. message CreateExecutionRequest { - bytes instanceHash = 1 [ - (gogoproto.customtype) = "github.com/mesg-foundation/engine/hash.Hash", - (gogoproto.nullable) = false - ]; string taskKey = 2; mesg.protobuf.Struct inputs = 3; repeated string tags = 4; diff --git a/sdk/backend.go b/sdk/backend.go index d9fe2aa0b..41ea47172 100644 --- a/sdk/backend.go +++ b/sdk/backend.go @@ -36,7 +36,7 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { service := servicesdk.NewBackend(appFactory, ownership) instance := instancesdk.NewBackend(appFactory) runner := runnersdk.NewBackend(appFactory, instance) - execution := executionsdk.NewBackend(appFactory, service, instance) + execution := executionsdk.NewBackend(appFactory, service, instance, runner) return &Backend{ Service: service, Ownership: ownership, diff --git a/sdk/execution/backend.go b/sdk/execution/backend.go index 9f2eac6ef..65913b3d2 100644 --- a/sdk/execution/backend.go +++ b/sdk/execution/backend.go @@ -14,6 +14,7 @@ import ( instancesdk "github.com/mesg-foundation/engine/sdk/instance" processsdk "github.com/mesg-foundation/engine/sdk/process" servicesdk "github.com/mesg-foundation/engine/sdk/service" + runnersdk "github.com/mesg-foundation/engine/sdk/runner" abci "github.com/tendermint/tendermint/abci/types" ) @@ -24,15 +25,17 @@ type Backend struct { storeKey *cosmostypes.KVStoreKey serviceBack *servicesdk.Backend instanceBack *instancesdk.Backend + runnerBack *runnersdk.Backend processSDK *processsdk.Process // TODO: to replace by process backend when process sdk is running on cosmos. } // NewBackend returns the backend of the execution sdk. -func NewBackend(appFactory *cosmos.AppFactory, serviceBack *servicesdk.Backend, instanceBack *instancesdk.Backend) *Backend { +func NewBackend(appFactory *cosmos.AppFactory, serviceBack *servicesdk.Backend, instanceBack *instancesdk.Backend, runnerBack *runnersdk.Backend) *Backend { backend := &Backend{ storeKey: cosmostypes.NewKVStoreKey(backendName), serviceBack: serviceBack, instanceBack: instanceBack, + runnerBack: runnerBack, } appBackendBasic := cosmos.NewAppModuleBasic(backendName) appBackend := cosmos.NewAppModule(appBackendBasic, backend.handler, backend.querier) @@ -85,7 +88,11 @@ func (s *Backend) querier(request cosmostypes.Request, path []string, req abci.R // Create creates a new execution from definition. func (s *Backend) Create(request cosmostypes.Request, msg msgCreateExecution) (*execution.Execution, error) { - inst, err := s.instanceBack.Get(request, msg.Request.InstanceHash) + run, err := s.runnerBack.Get(request, msg.Request.ExecutorHash) + if err != nil { + return nil, err + } + inst, err := s.instanceBack.Get(request, run.InstanceHash) if err != nil { return nil, err } @@ -103,7 +110,7 @@ func (s *Backend) Create(request cosmostypes.Request, msg msgCreateExecution) (* } exec := execution.New( msg.Request.ProcessHash, - msg.Request.InstanceHash, + run.InstanceHash, msg.Request.ParentHash, msg.Request.EventHash, msg.Request.StepID, From e78699be010ec6c09a1aaa951ca1bff671c35d84 Mon Sep 17 00:00:00 2001 From: krhubert Date: Wed, 20 Nov 2019 17:36:15 +0100 Subject: [PATCH 12/19] Refactor --- cosmos/client.go | 43 +++++++++++++++++------------- cosmos/module.go | 7 +++-- cosmos/type.go | 5 ++-- e2e/execution_test.go | 32 +++++++++++----------- orchestrator/mocks/ExecutionSDK.go | 30 ++++++++++++++------- orchestrator/orchestrator.go | 11 +++++--- orchestrator/type.go | 4 ++- sdk/execution/backend.go | 8 +++--- sdk/execution/sdk.go | 39 ++++++++++++++++----------- sdk/instance/backend.go | 2 +- sdk/ownership/backend.go | 2 +- sdk/runner/backend.go | 4 +-- sdk/service/backend.go | 4 +-- server/grpc/api/execution.go | 8 ++++-- x/xstrings/strings.go | 4 +-- 15 files changed, 121 insertions(+), 82 deletions(-) diff --git a/cosmos/client.go b/cosmos/client.go index d53bebe94..0043d5f02 100644 --- a/cosmos/client.go +++ b/cosmos/client.go @@ -133,32 +133,39 @@ func (c *Client) BuildAndBroadcastMsg(msg sdktypes.Msg, accName, accPassword str } // Stream subscribes to the provided query and returns the hash of the matching ressources. -func (c *Client) Stream(query string) (chan hash.Hash, error) { - subscriber := xstrings.RandAsciiLetters(8) +func (c *Client) Stream(ctx context.Context, query string) (chan hash.Hash, chan error, error) { + subscriber := xstrings.RandASCIILetters(8) eventStream, err := c.Subscribe(context.Background(), subscriber, query) if err != nil { - return nil, err + return nil, nil, err } hashC := make(chan hash.Hash) + errC := make(chan error) go func() { - for event := range eventStream { - ressHashes := event.Events[EventHashType] - if len(ressHashes) != 1 { - // or panic(err) - grpc api do not support - // return the errors on the stream for now - // so besieds logging the error, it not - // much we can do here. same belove - continue - } - - hash, err := hash.Decode(ressHashes[0]) - if err != nil { - continue + loop: + for { + select { + case event := <-eventStream: + tags := event.Events[EventHashType] + if len(tags) != 1 { + errC <- fmt.Errorf("event %s has %d tag(s), but only 1 is expected", EventHashType, len(tags)) + break + } + + hash, err := hash.Decode(tags[0]) + if err != nil { + errC <- err + } else { + hashC <- hash + } + case <-ctx.Done(): + break loop } - hashC <- hash } + close(errC) + close(hashC) c.Unsubscribe(context.Background(), subscriber, query) }() - return hashC, nil + return hashC, errC, nil } diff --git a/cosmos/module.go b/cosmos/module.go index b2ba37e0c..99b0dd533 100644 --- a/cosmos/module.go +++ b/cosmos/module.go @@ -33,7 +33,7 @@ type AppModule struct { } // Handler defines the core of the state transition function of an application. -type Handler func(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) +type Handler func(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) // Querier is responsible to answer to ABCI queries. type Querier func(request cosmostypes.Request, path []string, req abci.RequestQuery) (res interface{}, err error) @@ -107,7 +107,10 @@ func (m AppModule) NewHandler() cosmostypes.Handler { return func(request cosmostypes.Request, msg cosmostypes.Msg) cosmostypes.Result { hash, err := m.handler(request, msg) if err != nil { - return err.Result() + if errsdk, ok := err.(cosmostypes.Error); ok { + return errsdk.Result() + } + return cosmostypes.ErrInternal(err.Error()).Result() } events := request.EventManager().Events() diff --git a/cosmos/type.go b/cosmos/type.go index cfdc09059..2d0ffcc76 100644 --- a/cosmos/type.go +++ b/cosmos/type.go @@ -3,7 +3,6 @@ package cosmos import ( "fmt" - cosmostypes "github.com/cosmos/cosmos-sdk/types" sdktypes "github.com/cosmos/cosmos-sdk/types" ) @@ -12,8 +11,8 @@ const ( AttributeKeyHash = "hash" ) -// EventHashType -var EventHashType = cosmostypes.EventTypeMessage + "." + AttributeKeyHash +// EventHashType is a message with resource hash +var EventHashType = sdktypes.EventTypeMessage + "." + AttributeKeyHash func EventActionQuery(msgType string) string { return fmt.Sprintf("%s.%s='%s'", sdktypes.EventTypeMessage, sdktypes.AttributeKeyAction, msgType) diff --git a/e2e/execution_test.go b/e2e/execution_test.go index e7e4d8984..26297927f 100644 --- a/e2e/execution_test.go +++ b/e2e/execution_test.go @@ -15,15 +15,16 @@ import ( func testExecution(t *testing.T) { var executionHash hash.Hash - var execPingCompleted *execution.Execution + var execPing *execution.Execution + ctx := metadata.NewOutgoingContext(context.Background(), passmd) t.Run("stream", func(t *testing.T) { - t.Run("with nil filter", func(t *testing.T) { + t.Run("nil filter", func(t *testing.T) { _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{}) require.NoError(t, err) }) // TODO: no error are returned but it supposed to... - // t.Run("with not valid filter", func(t *testing.T) { + // t.Run("not valid filter", func(t *testing.T) { // t.Run("not found executor", func(t *testing.T) { // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ // Filter: &pb.StreamExecutionRequest_Filter{ @@ -50,7 +51,7 @@ func testExecution(t *testing.T) { // require.EqualError(t, err, "service \"test-service\" - task \"do-not-exist\" not found") // }) // }) - t.Run("working", func(t *testing.T) { + t.Run("good", func(t *testing.T) { stream, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ Filter: &pb.StreamExecutionRequest_Filter{ ExecutorHash: testRunnerHash, @@ -58,6 +59,7 @@ func testExecution(t *testing.T) { }) require.NoError(t, err) acknowledgement.WaitForStreamToBeReady(stream) + resp, err := client.ExecutionClient.Create(ctx, &pb.CreateExecutionRequest{ TaskKey: "ping", EventHash: hash.Int(1), @@ -82,24 +84,24 @@ func testExecution(t *testing.T) { require.Equal(t, execution.Status_InProgress, execPingInProgress.Status) }) t.Run("receive completed execution", func(t *testing.T) { - execPingCompleted, err = stream.Recv() + execPing, err = stream.Recv() require.NoError(t, err) - require.Equal(t, resp.Hash, execPingCompleted.Hash) - require.Equal(t, "ping", execPingCompleted.TaskKey) - require.Equal(t, execution.Status_Completed, execPingCompleted.Status) + require.Equal(t, resp.Hash, execPing.Hash) + require.Equal(t, "ping", execPing.TaskKey) + require.Equal(t, execution.Status_Completed, execPing.Status) }) }) }) t.Run("get", func(t *testing.T) { - execGet, err := client.ExecutionClient.Get(ctx, &pb.GetExecutionRequest{Hash: executionHash}) + exec, err := client.ExecutionClient.Get(ctx, &pb.GetExecutionRequest{Hash: executionHash}) require.NoError(t, err) - require.True(t, execGet.Equal(execPingCompleted)) + require.True(t, exec.Equal(execPing)) }) - // t.Run("list", func(t *testing.T) { - // execList, err := client.ExecutionClient.List(ctx, &pb.GetExecutionRequest{Hash: executionHash}) - // require.NoError(t, err) - // require.True(t, execGet.Equal(execPingCompleted)) - // }) + t.Run("list", func(t *testing.T) { + resp, err := client.ExecutionClient.List(ctx, &pb.ListExecutionRequest{}) + require.NoError(t, err) + require.Len(t, resp.Executions, 1) + }) } diff --git a/orchestrator/mocks/ExecutionSDK.go b/orchestrator/mocks/ExecutionSDK.go index cb360f645..8e8401ecc 100644 --- a/orchestrator/mocks/ExecutionSDK.go +++ b/orchestrator/mocks/ExecutionSDK.go @@ -3,6 +3,7 @@ package mocks import api "github.com/mesg-foundation/engine/protobuf/api" +import context "context" import execution "github.com/mesg-foundation/engine/execution" import hash "github.com/mesg-foundation/engine/hash" import mock "github.com/stretchr/testify/mock" @@ -58,25 +59,34 @@ func (_m *ExecutionSDK) Get(_a0 hash.Hash) (*execution.Execution, error) { return r0, r1 } -// Stream provides a mock function with given fields: req -func (_m *ExecutionSDK) Stream(req *api.StreamExecutionRequest) (chan *execution.Execution, error) { - ret := _m.Called(req) +// Stream provides a mock function with given fields: ctx, req +func (_m *ExecutionSDK) Stream(ctx context.Context, req *api.StreamExecutionRequest) (chan *execution.Execution, chan error, error) { + ret := _m.Called(ctx, req) var r0 chan *execution.Execution - if rf, ok := ret.Get(0).(func(*api.StreamExecutionRequest) chan *execution.Execution); ok { - r0 = rf(req) + if rf, ok := ret.Get(0).(func(context.Context, *api.StreamExecutionRequest) chan *execution.Execution); ok { + r0 = rf(ctx, req) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(chan *execution.Execution) } } - var r1 error - if rf, ok := ret.Get(1).(func(*api.StreamExecutionRequest) error); ok { - r1 = rf(req) + var r1 chan error + if rf, ok := ret.Get(1).(func(context.Context, *api.StreamExecutionRequest) chan error); ok { + r1 = rf(ctx, req) } else { - r1 = ret.Error(1) + if ret.Get(1) != nil { + r1 = ret.Get(1).(chan error) + } } - return r0, r1 + var r2 error + if rf, ok := ret.Get(2).(func(context.Context, *api.StreamExecutionRequest) error); ok { + r2 = rf(ctx, req) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index 384f3e22b..a2e06f8c6 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -1,6 +1,7 @@ package orchestrator import ( + "context" "fmt" "math/rand" @@ -32,8 +33,12 @@ func (s *Orchestrator) Start() error { if s.eventStream != nil || s.executionStream != nil { return fmt.Errorf("process orchestrator already running") } + s.eventStream = s.event.GetStream(nil) - executionStream, err := s.execution.Stream(&api.StreamExecutionRequest{ + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + executionStream, errC, err := s.execution.Stream(ctx, &api.StreamExecutionRequest{ Filter: &api.StreamExecutionRequest_Filter{ Statuses: []execution.Status{execution.Status_Completed}, }, @@ -49,10 +54,10 @@ func (s *Orchestrator) Start() error { case execution := <-s.executionStream: go s.execute(s.resultFilter(execution), execution, nil, execution.Outputs) go s.execute(s.dependencyFilter(execution), execution, nil, execution.Outputs) + case err := <-errC: + s.ErrC <- err } } - close(executionStream) - return nil } func (s *Orchestrator) eventFilter(event *event.Event) func(wf *process.Process, node *process.Process_Node) (bool, error) { diff --git a/orchestrator/type.go b/orchestrator/type.go index 13341d0e4..f7fdb18c7 100644 --- a/orchestrator/type.go +++ b/orchestrator/type.go @@ -1,6 +1,8 @@ package orchestrator import ( + "context" + "github.com/mesg-foundation/engine/execution" "github.com/mesg-foundation/engine/hash" "github.com/mesg-foundation/engine/process" @@ -12,7 +14,7 @@ import ( // ExecutionSDK execution interface needed for the orchestrator type ExecutionSDK interface { - Stream(req *api.StreamExecutionRequest) (chan *execution.Execution, error) + Stream(ctx context.Context, req *api.StreamExecutionRequest) (chan *execution.Execution, chan error, error) Get(hash hash.Hash) (*execution.Execution, error) Create(req *api.CreateExecutionRequest, accountName, accountPassword string) (*execution.Execution, error) } diff --git a/sdk/execution/backend.go b/sdk/execution/backend.go index 65913b3d2..bebf1c901 100644 --- a/sdk/execution/backend.go +++ b/sdk/execution/backend.go @@ -13,8 +13,8 @@ import ( "github.com/mesg-foundation/engine/protobuf/types" instancesdk "github.com/mesg-foundation/engine/sdk/instance" processsdk "github.com/mesg-foundation/engine/sdk/process" - servicesdk "github.com/mesg-foundation/engine/sdk/service" runnersdk "github.com/mesg-foundation/engine/sdk/runner" + servicesdk "github.com/mesg-foundation/engine/sdk/service" abci "github.com/tendermint/tendermint/abci/types" ) @@ -51,18 +51,18 @@ func (s *Backend) SetProcessSDK(processSDK *processsdk.Process) { s.processSDK = processSDK } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { switch msg := msg.(type) { case msgCreateExecution: exec, err := s.Create(request, msg) if err != nil { - return nil, cosmostypes.ErrInternal(err.Error()) + return nil, err } return exec.Hash, nil case msgUpdateExecution: exec, err := s.Update(request, msg) if err != nil { - return nil, cosmostypes.ErrInternal(err.Error()) + return nil, err } return exec.Hash, nil default: diff --git a/sdk/execution/sdk.go b/sdk/execution/sdk.go index 4203ed921..08edc3402 100644 --- a/sdk/execution/sdk.go +++ b/sdk/execution/sdk.go @@ -1,6 +1,7 @@ package executionsdk import ( + "context" "fmt" "github.com/mesg-foundation/engine/cosmos" @@ -89,32 +90,38 @@ func (s *SDK) List() ([]*execution.Execution, error) { } // Stream returns execution that matches given hash. -func (s *SDK) Stream(req *api.StreamExecutionRequest) (chan *execution.Execution, error) { +func (s *SDK) Stream(ctx context.Context, req *api.StreamExecutionRequest) (chan *execution.Execution, chan error, error) { if err := req.Filter.Validate(); err != nil { - return nil, err + return nil, nil, err } - stream, err := s.client.Stream(cosmos.EventModuleQuery(backendName)) + stream, serrC, err := s.client.Stream(ctx, cosmos.EventModuleQuery(backendName)) if err != nil { - return nil, err + return nil, nil, err } execC := make(chan *execution.Execution) + errC := make(chan error) go func() { - for hash := range stream { - exec, err := s.Get(hash) - if err != nil { - // or panic(err) - grpc api do not support - // return the errors on the stream for now - // so besieds logging the error, it not - // much we can do here. - continue - } - if req.Filter.Match(exec) { - execC <- exec + loop: + for { + select { + case hash := <-stream: + exec, err := s.Get(hash) + if err != nil { + errC <- err + } + if req.Filter.Match(exec) { + execC <- exec + } + case err := <-serrC: + errC <- err + case <-ctx.Done(): + break loop } } + close(errC) close(execC) }() - return execC, nil + return execC, errC, nil } diff --git a/sdk/instance/backend.go b/sdk/instance/backend.go index eb4b77e15..b94f6cf94 100644 --- a/sdk/instance/backend.go +++ b/sdk/instance/backend.go @@ -32,7 +32,7 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { errmsg := fmt.Sprintf("Unrecognized instance Msg type: %v", msg.Type()) return nil, cosmostypes.ErrUnknownRequest(errmsg) } diff --git a/sdk/ownership/backend.go b/sdk/ownership/backend.go index 8874122db..a0fc01a25 100644 --- a/sdk/ownership/backend.go +++ b/sdk/ownership/backend.go @@ -31,7 +31,7 @@ func NewBackend(appFactory *cosmos.AppFactory) *Backend { return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { errmsg := fmt.Sprintf("Unrecognized ownership Msg type: %v", msg.Type()) return nil, cosmostypes.ErrUnknownRequest(errmsg) } diff --git a/sdk/runner/backend.go b/sdk/runner/backend.go index c4e76eb5b..64ce754a3 100644 --- a/sdk/runner/backend.go +++ b/sdk/runner/backend.go @@ -35,7 +35,7 @@ func NewBackend(appFactory *cosmos.AppFactory, instanceBack *instancesdk.Backend return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { switch msg := msg.(type) { case msgCreateRunner: run, err := s.Create(request, &msg) @@ -45,7 +45,7 @@ func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (has return run.Hash, nil case msgDeleteRunner: if err := s.Delete(request, &msg); err != nil { - return nil, cosmostypes.ErrInternal(err.Error()) + return nil, err } return nil, nil default: diff --git a/sdk/service/backend.go b/sdk/service/backend.go index f482948bc..19988ced5 100644 --- a/sdk/service/backend.go +++ b/sdk/service/backend.go @@ -37,12 +37,12 @@ func NewBackend(appFactory *cosmos.AppFactory, ownerships *ownershipsdk.Backend) return backend } -func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, cosmostypes.Error) { +func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { switch msg := msg.(type) { case msgCreateService: srv, err := s.Create(request, &msg) if err != nil { - return nil, cosmostypes.ErrInternal(err.Error()) + return nil, err } return srv.Hash, nil default: diff --git a/server/grpc/api/execution.go b/server/grpc/api/execution.go index f3d38fcd1..896edfcb1 100644 --- a/server/grpc/api/execution.go +++ b/server/grpc/api/execution.go @@ -42,11 +42,13 @@ func (s *ExecutionServer) Get(ctx context.Context, req *api.GetExecutionRequest) // Stream returns stream of executions. func (s *ExecutionServer) Stream(req *api.StreamExecutionRequest, resp api.Execution_StreamServer) error { - stream, err := s.sdk.Execution.Stream(req) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + stream, errC, err := s.sdk.Execution.Stream(ctx, req) if err != nil { return err } - defer close(stream) if err := acknowledgement.SetStreamReady(resp); err != nil { return err @@ -58,6 +60,8 @@ func (s *ExecutionServer) Stream(req *api.StreamExecutionRequest, resp api.Execu if err := resp.Send(exec); err != nil { return err } + case err := <-errC: + return err case <-resp.Context().Done(): return resp.Context().Err() } diff --git a/x/xstrings/strings.go b/x/xstrings/strings.go index 854af7ccd..21025e14b 100644 --- a/x/xstrings/strings.go +++ b/x/xstrings/strings.go @@ -35,8 +35,8 @@ func SliceIndex(a []string, e string) int { var asciiletters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -// RandAsciiLetters generates random string from ascii letters. -func RandAsciiLetters(n int) string { +// RandASCIILetters generates random string from ascii letters. +func RandASCIILetters(n int) string { b := make([]byte, n) for i := range b { b[i] = asciiletters[rand.Intn(len(asciiletters))] From 8168750a62cff4c9966784f461890784775fec9f Mon Sep 17 00:00:00 2001 From: krhubert Date: Thu, 21 Nov 2019 09:34:20 +0100 Subject: [PATCH 13/19] Remove commented code + fix linter --- cosmos/type.go | 2 ++ e2e/execution_test.go | 28 ---------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/cosmos/type.go b/cosmos/type.go index 2d0ffcc76..864fd008d 100644 --- a/cosmos/type.go +++ b/cosmos/type.go @@ -14,10 +14,12 @@ const ( // EventHashType is a message with resource hash var EventHashType = sdktypes.EventTypeMessage + "." + AttributeKeyHash +// EventActionQuery returns tendermint query which matches given message type. func EventActionQuery(msgType string) string { return fmt.Sprintf("%s.%s='%s'", sdktypes.EventTypeMessage, sdktypes.AttributeKeyAction, msgType) } +// EventModuleQuery returns tendermint query which matches given module. func EventModuleQuery(module string) string { return fmt.Sprintf("%s.%s='%s'", sdktypes.EventTypeMessage, sdktypes.AttributeKeyModule, module) } diff --git a/e2e/execution_test.go b/e2e/execution_test.go index 26297927f..0433919ee 100644 --- a/e2e/execution_test.go +++ b/e2e/execution_test.go @@ -23,34 +23,6 @@ func testExecution(t *testing.T) { _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{}) require.NoError(t, err) }) - // TODO: no error are returned but it supposed to... - // t.Run("not valid filter", func(t *testing.T) { - // t.Run("not found executor", func(t *testing.T) { - // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ - // Filter: &pb.StreamExecutionRequest_Filter{ - // ExecutorHash: hash.Int(1), - // }, - // }) - // require.EqualError(t, err, "dwdw") - // }) - // t.Run("not found instance", func(t *testing.T) { - // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ - // Filter: &pb.StreamExecutionRequest_Filter{ - // InstanceHash: hash.Int(1), - // }, - // }) - // require.EqualError(t, err, "dwdw") - // }) - // t.Run("not found task key", func(t *testing.T) { - // _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ - // Filter: &pb.StreamExecutionRequest_Filter{ - // ExecutorHash: testRunnerHash, - // TaskKey: "do-not-exist", - // }, - // }) - // require.EqualError(t, err, "service \"test-service\" - task \"do-not-exist\" not found") - // }) - // }) t.Run("good", func(t *testing.T) { stream, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{ Filter: &pb.StreamExecutionRequest_Filter{ From 40e42a86d212dcb4d3f6af9bce6c8af34734e456 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 22 Nov 2019 12:07:32 +0700 Subject: [PATCH 14/19] remove ErrTxInCache in execution sdk --- sdk/execution/sdk.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sdk/execution/sdk.go b/sdk/execution/sdk.go index 08edc3402..16ee9d4a2 100644 --- a/sdk/execution/sdk.go +++ b/sdk/execution/sdk.go @@ -2,7 +2,6 @@ package executionsdk import ( "context" - "fmt" "github.com/mesg-foundation/engine/cosmos" "github.com/mesg-foundation/engine/execution" @@ -11,7 +10,6 @@ import ( instancesdk "github.com/mesg-foundation/engine/sdk/instance" runnersdk "github.com/mesg-foundation/engine/sdk/runner" servicesdk "github.com/mesg-foundation/engine/sdk/service" - "github.com/tendermint/tendermint/mempool" ) // SDK is the execution sdk. @@ -46,9 +44,6 @@ func (s *SDK) Create(req *api.CreateExecutionRequest, accountName, accountPasswo msg := newMsgCreateExecution(req, acc.GetAddress()) tx, err := s.client.BuildAndBroadcastMsg(msg, accountName, accountPassword) if err != nil { - if err == mempool.ErrTxInCache { - return nil, fmt.Errorf("execution already exists: %w", err) - } return nil, err } return s.Get(tx.Data) @@ -63,9 +58,6 @@ func (s *SDK) Update(req *api.UpdateExecutionRequest, accountName, accountPasswo msg := newMsgUpdateExecution(req, acc.GetAddress()) tx, err := s.client.BuildAndBroadcastMsg(msg, accountName, accountPassword) if err != nil { - if err == mempool.ErrTxInCache { - return nil, fmt.Errorf("execution already exists: %w", err) - } return nil, err } return s.Get(tx.Data) From a16664fe030e98dfe2317eb149b135cceea8ecc3 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 22 Nov 2019 13:14:00 +0700 Subject: [PATCH 15/19] Remove check on process in execution because process is not on cosmos yet (issue with new node sync) --- core/main.go | 4 +--- sdk/execution/backend.go | 19 ++++++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/core/main.go b/core/main.go index 10157b2bf..bf7cddfdd 100644 --- a/core/main.go +++ b/core/main.go @@ -149,7 +149,7 @@ func main() { // register the backend modules to the app factory. // TODO: this is a mandatory call so it should return a new types required by cosmos.NewApp - backend := enginesdk.NewBackend(appFactory) + enginesdk.NewBackend(appFactory) // init cosmos app app, err := cosmos.NewApp(appFactory) @@ -186,8 +186,6 @@ func main() { // init sdk sdk := enginesdk.New(client, kb, processDB, container, cfg.Name, strconv.Itoa(port), cfg.IpfsEndpoint) - // TODO: this is a hack and will be remove when process sdk is running on cosmos - backend.Execution.SetProcessSDK(sdk.Process) // start tendermint node logrus.WithField("module", "main").WithField("seeds", cfg.Tendermint.Config.P2P.Seeds).Info("starting tendermint node") diff --git a/sdk/execution/backend.go b/sdk/execution/backend.go index bebf1c901..e5b333f97 100644 --- a/sdk/execution/backend.go +++ b/sdk/execution/backend.go @@ -12,7 +12,6 @@ import ( "github.com/mesg-foundation/engine/protobuf/api" "github.com/mesg-foundation/engine/protobuf/types" instancesdk "github.com/mesg-foundation/engine/sdk/instance" - processsdk "github.com/mesg-foundation/engine/sdk/process" runnersdk "github.com/mesg-foundation/engine/sdk/runner" servicesdk "github.com/mesg-foundation/engine/sdk/service" abci "github.com/tendermint/tendermint/abci/types" @@ -26,7 +25,6 @@ type Backend struct { serviceBack *servicesdk.Backend instanceBack *instancesdk.Backend runnerBack *runnersdk.Backend - processSDK *processsdk.Process // TODO: to replace by process backend when process sdk is running on cosmos. } // NewBackend returns the backend of the execution sdk. @@ -45,12 +43,6 @@ func NewBackend(appFactory *cosmos.AppFactory, serviceBack *servicesdk.Backend, return backend } -// SetProcessSDK sets the process sdk. -// TODO: this is a hack and will be remove when process sdk is running on cosmos -func (s *Backend) SetProcessSDK(processSDK *processsdk.Process) { - s.processSDK = processSDK -} - func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (hash.Hash, error) { switch msg := msg.(type) { case msgCreateExecution: @@ -100,11 +92,12 @@ func (s *Backend) Create(request cosmostypes.Request, msg msgCreateExecution) (* if err != nil { return nil, err } - if !msg.Request.ProcessHash.IsZero() { - if _, err := s.processSDK.Get(msg.Request.ProcessHash); err != nil { - return nil, err - } - } + // TODO: to re-implement when process is on cosmos + // if !msg.Request.ProcessHash.IsZero() { + // if _, err := s.processSDK.Get(msg.Request.ProcessHash); err != nil { + // return nil, err + // } + // } if err := srv.RequireTaskInputs(msg.Request.TaskKey, msg.Request.Inputs); err != nil { return nil, err } From 6d37b56bac1220043c7b946de48b8e8cde546d49 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 22 Nov 2019 14:47:13 +0700 Subject: [PATCH 16/19] fix hash returned on error in runner backend handler function --- sdk/instance/backend.go | 2 -- sdk/runner/backend.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/instance/backend.go b/sdk/instance/backend.go index b94f6cf94..d36b5b626 100644 --- a/sdk/instance/backend.go +++ b/sdk/instance/backend.go @@ -50,7 +50,6 @@ func (s *Backend) querier(request cosmostypes.Request, path []string, req abci.R if err := codec.UnmarshalBinaryBare(req.Data, &f); err != nil { return nil, err } - return s.List(request, &f) case "exists": hash, err := hash.Decode(path[1]) @@ -58,7 +57,6 @@ func (s *Backend) querier(request cosmostypes.Request, path []string, req abci.R return nil, err } return s.Exists(request, hash) - default: return nil, errors.New("unknown instance query endpoint" + path[0]) } diff --git a/sdk/runner/backend.go b/sdk/runner/backend.go index 64ce754a3..338b21f9b 100644 --- a/sdk/runner/backend.go +++ b/sdk/runner/backend.go @@ -40,7 +40,7 @@ func (s *Backend) handler(request cosmostypes.Request, msg cosmostypes.Msg) (has case msgCreateRunner: run, err := s.Create(request, &msg) if err != nil { - return run.Hash, cosmostypes.ErrInternal(err.Error()) + return nil, cosmostypes.ErrInternal(err.Error()) } return run.Hash, nil case msgDeleteRunner: From 713b7a1bf00cde18ea875ccafc431c89d1a33dd0 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 22 Nov 2019 15:01:56 +0700 Subject: [PATCH 17/19] check for existing runner in runner sdk create --- sdk/runner/sdk.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdk/runner/sdk.go b/sdk/runner/sdk.go index 4a0adb3b1..b9c8a8dba 100644 --- a/sdk/runner/sdk.go +++ b/sdk/runner/sdk.go @@ -2,6 +2,7 @@ package runnersdk import ( "errors" + "fmt" cosmostypes "github.com/cosmos/cosmos-sdk/types" "github.com/mesg-foundation/engine/container" @@ -79,6 +80,11 @@ func (s *SDK) Create(req *api.CreateRunnerRequest, accountName, accountPassword InstanceHash: instanceHash, }) + // check runner already exists + if existingRun, _ := s.Get(expRunnerHash); existingRun != nil { + return nil, fmt.Errorf("runner %q already exists", existingRun.Hash) + } + // start the container imageHash, err := build(s.container, srv, s.ipfsEndpoint) if err != nil { From a552e8952a4132839b5c5579d2090c711e20d9f7 Mon Sep 17 00:00:00 2001 From: Nicolas Mahe Date: Fri, 22 Nov 2019 15:02:19 +0700 Subject: [PATCH 18/19] check for existing execution in execution sdk backend create function --- sdk/execution/backend.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/execution/backend.go b/sdk/execution/backend.go index e5b333f97..d1e26f6b8 100644 --- a/sdk/execution/backend.go +++ b/sdk/execution/backend.go @@ -112,10 +112,13 @@ func (s *Backend) Create(request cosmostypes.Request, msg msgCreateExecution) (* msg.Request.Tags, msg.Request.ExecutorHash, ) + store := request.KVStore(s.storeKey) + if store.Has(exec.Hash) { + return nil, fmt.Errorf("execution %q already exists", exec.Hash) + } if err := exec.Execute(); err != nil { return nil, err } - store := request.KVStore(s.storeKey) value, err := codec.MarshalBinaryBare(exec) if err != nil { return nil, err From adf0b716c38de4bd6e5d46cdfd654301b2277ed2 Mon Sep 17 00:00:00 2001 From: Anthony ESTEBE Date: Fri, 22 Nov 2019 17:00:29 +0700 Subject: [PATCH 19/19] fix lint --- e2e/execution_test.go | 2 +- e2e/testdata/test-service/main.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/execution_test.go b/e2e/execution_test.go index a28972604..ec21ff0f2 100644 --- a/e2e/execution_test.go +++ b/e2e/execution_test.go @@ -15,7 +15,7 @@ import ( func testExecution(t *testing.T) { var executionHash hash.Hash var execPing *execution.Execution - + t.Run("stream", func(t *testing.T) { t.Run("nil filter", func(t *testing.T) { _, err := client.ExecutionClient.Stream(context.Background(), &pb.StreamExecutionRequest{}) diff --git a/e2e/testdata/test-service/main.go b/e2e/testdata/test-service/main.go index 4261bdc5e..46f48517a 100644 --- a/e2e/testdata/test-service/main.go +++ b/e2e/testdata/test-service/main.go @@ -118,7 +118,7 @@ func main() { req.Result = &pb.UpdateExecutionRequest_Outputs{ Outputs: &types.Struct{ Fields: map[string]*types.Value{ - "pong": &types.Value{ + "pong": { Kind: &types.Value_StringValue{ StringValue: exec.Inputs.Fields["msg"].GetStringValue(), }, @@ -130,7 +130,7 @@ func main() { req.Result = &pb.UpdateExecutionRequest_Outputs{ Outputs: &types.Struct{ Fields: map[string]*types.Value{ - "res": &types.Value{ + "res": { Kind: &types.Value_NumberValue{ NumberValue: exec.Inputs.Fields["n"].GetNumberValue() + exec.Inputs.Fields["m"].GetNumberValue(), }, @@ -151,7 +151,7 @@ func main() { Key: exec.TaskKey + "_ok", Data: &types.Struct{ Fields: map[string]*types.Value{ - "msg": &types.Value{ + "msg": { Kind: &types.Value_StringValue{ StringValue: exec.TaskKey, },