Skip to content

Commit

Permalink
feature(callback): create callback plugin
Browse files Browse the repository at this point in the history
Signed-off-by: Thomas Bétrancourt <thomas@betrancourt.net>
  • Loading branch information
rclsilver committed Aug 17, 2022
1 parent 4ef80c2 commit 79bd458
Show file tree
Hide file tree
Showing 25 changed files with 1,390 additions and 21 deletions.
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,8 @@ dist/

# VScode extension
**/*.vsix

# Related to tests
docker-compose.override.yaml
hack/test.env
report.xml
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,10 @@ release-utask-lib:
test:
# moving to another location to go get some packages, otherwise it will include those packages as dependencies in go.mod
cd ${HOME} && go install github.com/jstemmer/go-junit-report/v2@latest
GO111MODULE=on DEV=true bash hack/test.sh ${TEST_CMD} 2>&1 | go-junit-report -set-exit-code > report.xml
GO111MODULE=on DEV=true bash hack/test.sh ${TEST_CMD} 2>&1 | go-junit-report | tee report.xml

test-dev:
GO111MODULE=on DEV=true bash hack/test.sh ${TEST_CMD} 2>&1

test-travis:
# moving to another location to go get some packages, otherwise it will include those packages as dependencies in go.mod
Expand Down
23 changes: 12 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -506,17 +506,18 @@ All the strategies available are:

Browse [builtin actions](./pkg/plugins/builtin)

| Plugin name | Description | Documentation |
| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------ |
| **`echo`** | Print out a pre-determined result | [Access plugin doc](./pkg/plugins/builtin/echo/README.md) |
| **`http`** | Make an http request | [Access plugin doc](./pkg/plugins/builtin/http/README.md) |
| **`subtask`** | Spawn a new task on µTask | [Access plugin doc](./pkg/plugins/builtin/subtask/README.md) |
| **`notify`** | Dispatch a notification over a registered channel | [Access plugin doc](./pkg/plugins/builtin/notify/README.md) |
| **`apiovh`** | Make a signed call on OVH's public API (requires credentials retrieved from configstore, containing the fields `endpoint`, `appKey`, `appSecret`, `consumerKey`, more info [here](https://docs.ovh.com/gb/en/customer/first-steps-with-ovh-api/)) | [Access plugin doc](./pkg/plugins/builtin/apiovh/README.md) |
| **`ssh`** | Connect to a remote system and run commands on it | [Access plugin doc](./pkg/plugins/builtin/ssh/README.md) |
| **`email`** | Send an email | [Access plugin doc](./pkg/plugins/builtin/email/README.md) |
| **`ping`** | Send a ping to an hostname *Warn: This plugin will keep running until the count is done* | [Access plugin doc](./pkg/plugins/builtin/ping/README.md) |
| **`script`** | Execute a script under `scripts` folder | [Access plugin doc](./pkg/plugins/builtin/script/README.md) |
| Plugin name | Description | Documentation |
| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- |
| **`echo`** | Print out a pre-determined result | [Access plugin doc](./pkg/plugins/builtin/echo/README.md) |
| **`http`** | Make an http request | [Access plugin doc](./pkg/plugins/builtin/http/README.md) |
| **`subtask`** | Spawn a new task on µTask | [Access plugin doc](./pkg/plugins/builtin/subtask/README.md) |
| **`notify`** | Dispatch a notification over a registered channel | [Access plugin doc](./pkg/plugins/builtin/notify/README.md) |
| **`apiovh`** | Make a signed call on OVH's public API (requires credentials retrieved from configstore, containing the fields `endpoint`, `appKey`, `appSecret`, `consumerKey`, more info [here](https://docs.ovh.com/gb/en/customer/first-steps-with-ovh-api/)) | [Access plugin doc](./pkg/plugins/builtin/apiovh/README.md) |
| **`ssh`** | Connect to a remote system and run commands on it | [Access plugin doc](./pkg/plugins/builtin/ssh/README.md) |
| **`email`** | Send an email | [Access plugin doc](./pkg/plugins/builtin/email/README.md) |
| **`ping`** | Send a ping to an hostname *Warn: This plugin will keep running until the count is done* | [Access plugin doc](./pkg/plugins/builtin/ping/README.md) |
| **`script`** | Execute a script under `scripts` folder | [Access plugin doc](./pkg/plugins/builtin/script/README.md) |
| **`callback`** | Use callbacks to manage your tasks life-cycle | [Access plugin doc](./pkg/plugins/builtin/callback/README.md) |

#### PreHooks <a name="preHooks"></a>

Expand Down
40 changes: 40 additions & 0 deletions api/bind.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package api

import (
"reflect"

"github.com/gin-gonic/gin"
"github.com/loopfz/gadgeto/tonic"
)

// bodyBindHook is a wrapper around the default binding hook of tonic.
// It adds the possibility to bind a specific field in an object rather than
// unconditionally binding the whole object.
func bodyBindHook(c *gin.Context, v interface{}) error {
val := reflect.ValueOf(v)
typ := reflect.TypeOf(v).Elem()

for i := 0; i < typ.NumField(); i++ {
ft := typ.Field(i)
if _, ok := ft.Tag.Lookup("body"); !ok {
continue
}
flt := ft.Type
var fv reflect.Value
if flt.Kind() == reflect.Map {
fv = reflect.New(flt)
} else {
fv = reflect.New(flt.Elem())
}
if err := tonic.DefaultBindingHook(c, fv.Interface()); err != nil {
return err
}
if flt.Kind() == reflect.Map {
val.Elem().Field(i).Set(fv.Elem())
} else {
val.Elem().Field(i).Set(fv)
}
}

return tonic.DefaultBindingHook(c, v)
}
54 changes: 54 additions & 0 deletions api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"syscall"

"github.com/gin-gonic/gin"
"github.com/juju/errors"
"github.com/loopfz/gadgeto/tonic"
"github.com/loopfz/gadgeto/tonic/utils/jujerr"
"github.com/loopfz/gadgeto/zesty"
Expand All @@ -21,11 +22,28 @@ import (

"github.com/ovh/utask"
"github.com/ovh/utask/api/handler"
"github.com/ovh/utask/db"
"github.com/ovh/utask/models/resolution"
"github.com/ovh/utask/models/task"
"github.com/ovh/utask/pkg/auth"
)

type PluginRoute struct {
Secured bool
Maintenance bool
Path string
Method string
Infos []fizz.OperationOption
Handlers []gin.HandlerFunc
}

type PluginRouterGroup struct {
Path string
Name string
Description string
Routes []PluginRoute
}

// Server wraps the http handler that exposes a REST API to control
// the task orchestration engine
type Server struct {
Expand All @@ -37,6 +55,7 @@ type Server struct {
editorPathPrefix string
maxBodyBytes int64
customMiddlewares []gin.HandlerFunc
pluginRoutes []PluginRouterGroup
}

// NewServer returns a new Server
Expand Down Expand Up @@ -132,6 +151,17 @@ func (s *Server) Handler(ctx context.Context) http.Handler {
return s.httpHandler
}

// RegisterPluginRoutes allows plugins to register custom routes
func (s *Server) RegisterPluginRoutes(group PluginRouterGroup) error {
for _, route := range group.Routes {
if len(route.Handlers) == 0 {
return errors.NewNotImplemented(nil, "found route without handler")
}
}
s.pluginRoutes = append(s.pluginRoutes, group)
return nil
}

func generateBaseHref(pathPrefix, uri string) string {
// UI requires to have a trailing slash at the end
return path.Join(pathPrefix, uri) + "/"
Expand Down Expand Up @@ -199,6 +229,7 @@ func (s *Server) build(ctx context.Context) {

tonic.SetErrorHook(jujerr.ErrHook)
tonic.SetBindHook(yamlBindHook(s.maxBodyBytes))
tonic.SetBindHook(bodyBindHook)
tonic.SetRenderHook(yamljsonRenderHook, "application/json")

authRoutes := router.Group("/", "x-misc", "Misc authenticated routes", s.authMiddleware)
Expand Down Expand Up @@ -458,6 +489,26 @@ func (s *Server) build(ctx context.Context) {
},
tonic.Handler(Stats, 200))

// plugin routes
for _, p := range s.pluginRoutes {
group := router.Group(p.Path, p.Name, p.Description)

for _, r := range p.Routes {
routeHandlers := []gin.HandlerFunc{}

if r.Maintenance {
routeHandlers = append(routeHandlers, maintenanceMode)
}
if r.Secured {
routeHandlers = append(routeHandlers, s.authMiddleware)
}

routeHandlers = append(routeHandlers, r.Handlers...)

group.Handle(r.Path, r.Method, r.Infos, routeHandlers...)
}
}

s.httpHandler = router
}
}
Expand Down Expand Up @@ -531,6 +582,9 @@ func keyRotate(c *gin.Context) error {
if err != nil {
return err
}
if err := db.CallKeyRotations(dbp); err != nil {
return err
}
if err := task.RotateTasks(dbp); err != nil {
return err
}
Expand Down
6 changes: 5 additions & 1 deletion cmd/utask/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,15 @@ var rootCmd = &cobra.Command{
server = api.NewServer()
server.WithGroupAuth(defaultAuthHandler)

service := &plugins.Service{Store: store, Server: server}

for _, err := range []error{
// register builtin initializers
builtin.RegisterInit(service),
// register builtin executors
builtin.Register(),
// run custom initialization code built as *.so plugins
plugins.InitializersFromFolder(utask.FInitializersFolder, &plugins.Service{Store: store, Server: server}),
plugins.InitializersFromFolder(utask.FInitializersFolder, service),
// load custom executors built as *.so plugins
plugins.ExecutorsFromFolder(utask.FPluginFolder),
// load the functions
Expand Down
37 changes: 37 additions & 0 deletions db/encrypt.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package db

import (
"sync"

"github.com/loopfz/gadgeto/zesty"
)

type KeyRotationCallback func(dbp zesty.DBProvider) error

var (
keyRotationsCb []KeyRotationCallback
keyRotationsCbMu sync.Mutex
)

// RegisterKeyRotations registers a callback which will be called during the encrypt
// keys rotations
func RegisterKeyRotations(cb KeyRotationCallback) {
keyRotationsCbMu.Lock()
defer keyRotationsCbMu.Unlock()

keyRotationsCb = append(keyRotationsCb, cb)
}

// CallKeyRotations calls registered callbacks to rotate the encryption keys
func CallKeyRotations(dbp zesty.DBProvider) error {
keyRotationsCbMu.Lock()
defer keyRotationsCbMu.Unlock()

for _, cb := range keyRotationsCb {
if err := cb(dbp); err != nil {
return err
}
}

return nil
}
5 changes: 5 additions & 0 deletions db/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ var schema = []tableModel{
{runnerinstance.Instance{}, "runner_instance", []string{"id"}, true},
}

// RegisterTableModel registers a new table model
func RegisterTableModel(model interface{}, name string, keys []string, autoinc bool) {
schema = append(schema, tableModel{model, name, keys, autoinc})
}

// Init takes a connection string and a configuration struct
// and registers a new postgres DB connection in zesty,
// under utask.DBName -> accessible from api handlers and engine collectors
Expand Down
2 changes: 1 addition & 1 deletion db/migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
)

const (
expectedVersion = "v1.19.0-migration007"
expectedVersion = "v1.20.0-migration008"
)

var (
Expand Down
27 changes: 27 additions & 0 deletions engine/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/stretchr/testify/require"

"github.com/ovh/utask"
"github.com/ovh/utask/api"
"github.com/ovh/utask/db"
"github.com/ovh/utask/db/pgjuju"
"github.com/ovh/utask/engine"
Expand All @@ -34,6 +35,8 @@ import (
"github.com/ovh/utask/models/task"
"github.com/ovh/utask/models/tasktemplate"
"github.com/ovh/utask/pkg/now"
"github.com/ovh/utask/pkg/plugins"
plugincallback "github.com/ovh/utask/pkg/plugins/builtin/callback"
"github.com/ovh/utask/pkg/plugins/builtin/echo"
"github.com/ovh/utask/pkg/plugins/builtin/script"
pluginsubtask "github.com/ovh/utask/pkg/plugins/builtin/subtask"
Expand All @@ -52,6 +55,13 @@ func TestMain(m *testing.M) {
store := configstore.DefaultStore
store.InitFromEnvironment()

server := api.NewServer()
service := &plugins.Service{Store: store, Server: server}

if err := plugincallback.Init.Init(service); err != nil {
panic(err)
}

if err := db.Init(store); err != nil {
panic(err)
}
Expand All @@ -76,6 +86,7 @@ func TestMain(m *testing.M) {
step.RegisterRunner(echo.Plugin.PluginName(), echo.Plugin)
step.RegisterRunner(script.Plugin.PluginName(), script.Plugin)
step.RegisterRunner(pluginsubtask.Plugin.PluginName(), pluginsubtask.Plugin)
step.RegisterRunner(plugincallback.Plugin.PluginName(), plugincallback.Plugin)

os.Exit(m.Run())
}
Expand Down Expand Up @@ -1204,3 +1215,19 @@ func TestResolveSubTask(t *testing.T) {
}
assert.Equal(t, resolution.StateDone, res.State)
}

func TestResolveCallback(t *testing.T) {
res, err := createResolution("callback.yaml", map[string]interface{}{}, nil)
require.NoError(t, err)

res, err = runResolution(res)
require.NoError(t, err)
require.NotNil(t, res)

// check steps state
assert.Equal(t, res.Steps["createCallback"].State, step.StateDone)
assert.Equal(t, res.Steps["waitCallback"].State, step.StateWaiting)

// callback has been created, waiting for its resolution
assert.Equal(t, resolution.StateWaiting, res.State)
}
38 changes: 38 additions & 0 deletions engine/templates_tests/callback.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: callbackTemplate
description: Template that test the callbacks
title_format: "[test] callback template test"
variables:
- name: successField
value: 'success'
steps:
createCallback:
description: create a callback
action:
type: callback
configuration:
action: create
schema: |-
{
"$schema": "http://json-schema.org/schema#",
"type": "object",
"additionalProperties": false,
"required": ["{{evalCache `successField`}}"],
"properties": {
"{{evalCache `successField`}}": {
"type": "boolean"
}
}
}
waitCallback:
dependencies:
- createCallback
description: everything is OK
action:
type: callback
configuration:
action: wait
id: '{{.step.createCallback.output.id}}'

result_format:
foo: "{{.step.waitCallback.output.success}}"
Loading

0 comments on commit 79bd458

Please sign in to comment.