Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactoring pipeline and engine #100

Merged
merged 2 commits into from
Mar 30, 2015
Merged

Refactoring pipeline and engine #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c270154
Refactoring pipeline and engine
takahi-i Mar 26, 2015
9a4ee53
Enhance a TODO comment
takahi-i Mar 30, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions config/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ func getStageTypeModuleName(stageType string) string {
return strings.ToLower(stageType)
}

func Parse(configData *map[interface{}]interface{}) (*pipelines.Pipeline, error) {
func Parse(configData *map[interface{}]interface{}) (*pipelines.Resources, error) {
envs := NewEnvVariables()
return ParseWithSpecifiedEnvs(configData, envs)
}

// TODO: need refactoring
// TODO: make parser process a struct (for simplifying redundant functions and reducing the number of function parameters)
func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
envs *EnvVariables) (*pipelines.Pipeline, error) {
envs *EnvVariables) (*pipelines.Resources, error) {
// parse service block
serviceOps, ok := (*configData)["service"].(map[interface{}]interface{})
var repoService services.Service
Expand Down Expand Up @@ -93,9 +93,7 @@ func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
}

// parse pipeline block
var pipeline *pipelines.Pipeline = &pipelines.Pipeline{
Reporter: messenger, RepoService: repoService, Cleanup: cleanup,
}
var pipeline *pipelines.Pipeline = &pipelines.Pipeline{}

pipelineData, ok := (*configData)["pipeline"].([]interface{})
if ok == false {
Expand All @@ -108,7 +106,9 @@ func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
pipeline.AddStage(stageItem.Value.(stages.Stage))
}
return pipeline, nil
var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}

return resources, nil
}

func mapMessenger(messengerMap map[interface{}]interface{}, envs *EnvVariables) (messengers.Messenger, error) {
Expand Down
22 changes: 11 additions & 11 deletions config/parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import (

func TestParseFromFile(t *testing.T) {
configData := ReadConfig("../tests/fixtures/pipeline.yml")
pipeline, err := Parse(configData)
actual := pipeline.Stages.Front().Value.(*stages.CommandStage).Command
resources, err := Parse(configData)
actual := resources.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello, world\"", actual)
assert.Nil(t, err)
}
Expand Down Expand Up @@ -60,10 +60,10 @@ func TestParseConfWithChildren(t *testing.T) {
type: command
command: echo "hello, world, command_stage_3_group_1"`))
result, err := Parse(configData)
assert.Equal(t, 1, result.Size())
assert.Equal(t, 1, result.Pipeline.Size())
assert.Nil(t, err)

childStages := result.Stages.Front().Value.(stages.Stage).GetChildStages()
childStages := result.Pipeline.Stages.Front().Value.(stages.Stage).GetChildStages()
assert.Equal(t, 2, childStages.Len())
}

Expand All @@ -73,7 +73,7 @@ func TestParseConfDefaultStageTypeIsCommand(t *testing.T) {
command: echo "hello, world"
`))
result, err := Parse(configData)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello, world\"", actual)
assert.Nil(t, err)
}
Expand All @@ -86,7 +86,7 @@ func TestParseConfWithDirectory(t *testing.T) {
directory: /usr/local
`))
result, err := Parse(configData)
actual := result.Stages.Front().Value.(*stages.CommandStage).Directory
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Directory
assert.Nil(t, err)
assert.Equal(t, "/usr/local", actual)
}
Expand All @@ -98,7 +98,7 @@ func TestParseConfWithShellScriptStage(t *testing.T) {
file: ../stages/test_sample.sh
`))
result, err := Parse(configData)
actual := result.Stages.Front().Value.(*stages.ShellScriptStage).File
actual := result.Pipeline.Stages.Front().Value.(*stages.ShellScriptStage).File
assert.Equal(t, "../stages/test_sample.sh", actual)
assert.Nil(t, err)
}
Expand Down Expand Up @@ -179,7 +179,7 @@ func TestParseConfWithEnvVariable(t *testing.T) {
envs := NewEnvVariables()
envs.Add("USER_NAME", "takahi-i")
result, err := ParseWithSpecifiedEnvs(configData, envs)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello $USER_NAME\"", actual)
assert.Nil(t, err)
}
Expand All @@ -205,7 +205,7 @@ func TestParseConfWithNoExistEnvVariable(t *testing.T) {
envs := NewEnvVariables()
envs.Add("USER_NAME", "takahi-i")
result, err := ParseWithSpecifiedEnvs(configData, envs)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello $NO_SUCH_A_ENV_VARIABLE\"", actual) // NOTE: No env variable name is shown when there is no env variable
assert.Nil(t, err)
}
Expand Down Expand Up @@ -240,8 +240,8 @@ func TestParseConfigWithDeprecatedProperties(t *testing.T) {
command: echo "hello, world"
`))
result, err := Parse(configData)
assert.Equal(t, 1, result.Size())
assert.Equal(t, 1, result.Pipeline.Size())
assert.Nil(t, err)
actual := result.Stages.Front().Value.(*stages.CommandStage).Command
actual := result.Pipeline.Stages.Front().Value.(*stages.CommandStage).Command
assert.Equal(t, "echo \"hello, world\"", actual)
}
10 changes: 5 additions & 5 deletions engine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ import (
)

type Engine struct {
Pipeline *pipelines.Pipeline
Opts *config.Opts
Resources *pipelines.Resources
MonitorCh *chan stages.Mediator
Opts *config.Opts
}

type Result struct {
Expand All @@ -45,8 +45,8 @@ func (r *Result) IsSucceeded() bool {
}

func (e *Engine) RunOnce() *Result {
pipe_result := e.executePipeline(e.Pipeline, "pipeline")
cleanup_result := e.executePipeline(e.Pipeline.Cleanup, "cleanup")
pipe_result := e.executePipeline(e.Resources.Pipeline, "pipeline")
cleanup_result := e.executePipeline(e.Resources.Cleanup, "cleanup")
return &Result{Pipeline: &pipe_result, Cleanup: &cleanup_result}
}

Expand Down Expand Up @@ -92,7 +92,7 @@ func (e *Engine) ExecuteStage(stage stages.Stage) {
result = false
}
log.Debugf("Stage execution results: %+v, %+v", stage.GetStageName(), result)
e.Pipeline.ReportStageResult(stage, result)
e.Resources.ReportStageResult(stage, result)

mediator := stages.Mediator{States: make(map[string]string)}
mediator.States[stage.GetStageName()] = fmt.Sprintf("%v", result)
Expand Down
80 changes: 44 additions & 36 deletions engine/engine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func execute(stage stages.Stage) stages.Mediator {
mon := make(chan stages.Mediator)
e := &Engine{
MonitorCh: &mon,
Pipeline: &pipelines.Pipeline{
Resources: &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
},
}
Expand Down Expand Up @@ -102,15 +102,16 @@ func execute(stage stages.Stage) stages.Mediator {
}

func TestRunOnce(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("echo baz"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("echo baz"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand All @@ -122,14 +123,15 @@ func TestRunOnce(t *testing.T) {
}

func TestRunOnceWithShellScriptStage(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createShellScriptStage("foobar-shell", "../stages/test_sample.sh"))
resources.Pipeline.AddStage(createShellScriptStage("foobar-shell", "../stages/test_sample.sh"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand All @@ -141,17 +143,18 @@ func TestRunOnceWithShellScriptStage(t *testing.T) {
}

func TestRunOnceWithOptsOffStopOnAnyFailure(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("thisiserrorcommand"))
pipeline.AddStage(createCommandStage("echo foobar2"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("thisiserrorcommand"))
resources.Pipeline.AddStage(createCommandStage("echo foobar2"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{StopOnAnyFailure: false}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -162,17 +165,18 @@ func TestRunOnceWithOptsOffStopOnAnyFailure(t *testing.T) {
}

func TestRunOnceWithOptsOnStopOnAnyFailure(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("thisiserrorcommand"))
pipeline.AddStage(createCommandStage("echo foobar2"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("thisiserrorcommand"))
resources.Pipeline.AddStage(createCommandStage("echo foobar2"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{StopOnAnyFailure: true}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -186,17 +190,18 @@ func TestRunOnceWithOptsOnStopOnAnyFailure(t *testing.T) {
}

func TestRunOnceWithOnlyIfFailure(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -lt 1"))
pipeline.AddStage(createCommandStageWithName("second", "echo second"))
pipeline.AddStage(createCommandStageWithName("third", "echo third"))
resources.Pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -lt 1"))
resources.Pipeline.AddStage(createCommandStageWithName("second", "echo second"))
resources.Pipeline.AddStage(createCommandStageWithName("third", "echo third"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -212,17 +217,18 @@ func TestRunOnceWithOnlyIfFailure(t *testing.T) {
}

func TestRunOnceWithOnlyIfSuccess(t *testing.T) {
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: &pipelines.Pipeline{},
}
pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -eq 1"))
pipeline.AddStage(createCommandStageWithName("second", "echo second"))
pipeline.AddStage(createCommandStageWithName("third", "echo third"))
resources.Pipeline.AddStage(createCommandStageWithOnlyIf("first", "echo first", "test 1 -eq 1"))
resources.Pipeline.AddStage(createCommandStageWithName("second", "echo second"))
resources.Pipeline.AddStage(createCommandStageWithName("third", "echo third"))
monitorCh := make(chan stages.Mediator)
o := &config.Opts{}
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
Opts: o,
}
Expand All @@ -241,16 +247,17 @@ func TestRunOnceWithCleanup(t *testing.T) {
cleanup := &pipelines.Pipeline{}
cleanup.AddStage(createCommandStage("echo cleanup"))
cleanup.AddStage(createCommandStage("echo baz"))
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: cleanup,
}

pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("echo baz"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("echo baz"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand All @@ -264,16 +271,17 @@ func TestRunOnceWithCleanup(t *testing.T) {
func TestRunOnceWithFailedCleanup(t *testing.T) {
cleanup := &pipelines.Pipeline{}
cleanup.AddStage(createCommandStage("nosuchacommand"))
pipeline := &pipelines.Pipeline{
resources := &pipelines.Resources{
Reporter: &messengers.FakeMessenger{},
Pipeline: &pipelines.Pipeline{},
Cleanup: cleanup,
}

pipeline.AddStage(createCommandStage("echo foobar"))
pipeline.AddStage(createCommandStage("echo baz"))
resources.Pipeline.AddStage(createCommandStage("echo foobar"))
resources.Pipeline.AddStage(createCommandStage("echo baz"))
monitorCh := make(chan stages.Mediator)
engine := &Engine{
Pipeline: pipeline,
Resources: resources,
MonitorCh: &monitorCh,
}
result := engine.RunOnce()
Expand Down
10 changes: 7 additions & 3 deletions pipelines/pipeline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,17 @@ import (
)

type Pipeline struct {
Stages list.List
Stages list.List
}

type Resources struct {
Pipeline *Pipeline
Cleanup *Pipeline
Reporter messengers.Messenger
RepoService services.Service
Cleanup *Pipeline
}

func (self *Pipeline) ReportStageResult(stage stages.Stage, result bool) {
func (self *Resources) ReportStageResult(stage stages.Stage, result bool) {
name := stage.GetStageName()
self.Reporter.Post(
fmt.Sprintf("Stage execution results: %+v, %+v", name, result))
Expand Down
4 changes: 2 additions & 2 deletions pipelines/pipeline_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (mock *MockMessenger) Post(msg string) bool {

func TestReportStageResult(t *testing.T) {
mock := &MockMessenger{}
p := Pipeline{
p := Resources{
Reporter: mock,
}

Expand All @@ -76,7 +76,7 @@ func TestReportStageResult(t *testing.T) {

func TestReportStageResultWithFullOutput(t *testing.T) {
mock := &MockMessenger{}
p := Pipeline{
p := Resources{
Reporter: mock,
}

Expand Down
Loading