Skip to content

Commit

Permalink
fix: misaligned nodes when running tasks from a schedule through enha…
Browse files Browse the repository at this point in the history
…ncement by adding new route and handler for running schedules

- Introduced a new POST endpoint '/:id/run' in the router to trigger the execution of a schedule.
- Implemented the PostScheduleRun function to handle the execution logic, including validation of input parameters and scheduling tasks.
- Refactored the PostSpiderRun function to streamline the scheduling process by directly calling the admin service.
- Updated the FsFileInfo struct to clarify the children field description.
- Modified the Task model to make NodeIds field optional in JSON serialization.
  • Loading branch information
tikazyq committed Dec 21, 2024
1 parent b88b41a commit 41e973a
Show file tree
Hide file tree
Showing 6 changed files with 63 additions and 6 deletions.
5 changes: 5 additions & 0 deletions core/controllers/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ func InitRoutes(app *gin.Engine) (err error) {
Path: "/:id/disable",
HandlerFunc: PostScheduleDisable,
},
{
Method: http.MethodPost,
Path: "/:id/run",
HandlerFunc: PostScheduleRun,
},
}...))
RegisterController(groups.AuthGroup, "/spiders", NewController[models.Spider]([]Action{
{
Expand Down
48 changes: 48 additions & 0 deletions core/controllers/schedule.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package controllers

import (
errors2 "errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/schedule"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"
)
Expand Down Expand Up @@ -116,3 +118,49 @@ func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) {
HandleSuccess(c)
}
}

func PostScheduleRun(c *gin.Context) {
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
HandleErrorBadRequest(c, err)
return
}

// options
var opts interfaces.SpiderRunOptions
if err := c.ShouldBindJSON(&opts); err != nil {
HandleErrorInternalServerError(c, err)
return
}
if opts.ScheduleId.IsZero() {
opts.ScheduleId = id
}

// schedule
sch, err := service.NewModelService[models.Schedule]().GetById(id)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}

// spider
s, err := service.NewModelService[models.Spider]().GetById(sch.SpiderId)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}

// user
if u := GetUserFromContext(c); u != nil {
opts.UserId = u.GetId()
}

// schedule tasks
taskIds, err := admin.GetSpiderAdminService().Schedule(s.Id, &opts)
if err != nil {
HandleErrorInternalServerError(c, err)
return
}

HandleSuccessWithData(c, taskIds)
}
6 changes: 2 additions & 4 deletions core/controllers/spider.go
Original file line number Diff line number Diff line change
Expand Up @@ -660,10 +660,8 @@ func PostSpiderRun(c *gin.Context) {
opts.UserId = u.GetId()
}

adminSvc := admin.GetSpiderAdminService()

// schedule
taskIds, err := adminSvc.Schedule(id, &opts)
// schedule tasks
taskIds, err := admin.GetSpiderAdminService().Schedule(id, &opts)
if err != nil {
HandleErrorInternalServerError(c, err)
return
Expand Down
6 changes: 6 additions & 0 deletions core/controllers/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,12 @@ func PostTaskRestart(c *gin.Context) {
Priority: t.Priority,
}

// normalize options for tasks with assigned node
if !t.NodeId.IsZero() {
opts.NodeIds = []primitive.ObjectID{t.NodeId}
opts.Mode = constants.RunTypeSelectedNodes
}

// user
if u := GetUserFromContext(c); u != nil {
opts.UserId = u.Id
Expand Down
2 changes: 1 addition & 1 deletion core/entity/fs_file_info.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ type FsFileInfo struct {
Extension string `json:"extension"` // file extension
IsDir bool `json:"is_dir"` // whether it is directory
FileSize int64 `json:"file_size"` // file size (bytes)
Children []interfaces.FsFileInfo `json:"children"` // children for sub-directory
Children []interfaces.FsFileInfo `json:"children"` // children for subdirectory
ModTime time.Time `json:"mod_time"` // modification time
Mode os.FileMode `json:"mode"` // file mode
Hash string `json:"hash"` // file hash
Expand Down
2 changes: 1 addition & 1 deletion core/models/models/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ type Task struct {
ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"`
Type string `json:"type" bson:"type"`
Mode string `json:"mode" bson:"mode"`
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"`
Priority int `json:"priority" bson:"priority"`
NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"-"`
Stat *TaskStat `json:"stat,omitempty" bson:"-"`
Spider *Spider `json:"spider,omitempty" bson:"-"`
Schedule *Schedule `json:"schedule,omitempty" bson:"-"`
Expand Down

0 comments on commit 41e973a

Please sign in to comment.