Skip to content

Commit

Permalink
Synthesize a command for a Run (#4499)
Browse files Browse the repository at this point in the history
  • Loading branch information
Greg Soltis authored Apr 10, 2023
1 parent 823eb41 commit 99a8b68
Show file tree
Hide file tree
Showing 9 changed files with 240 additions and 81 deletions.
2 changes: 2 additions & 0 deletions cli/integration_tests/single_package/run-summary.t
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@ Check
[
"attempted",
"cached",
"command",
"endTime",
"exitCode",
"failed",
"repoPath",
"startTime",
"success"
]
Expand Down
18 changes: 5 additions & 13 deletions cli/internal/run/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,6 @@ import (
"github.com/pkg/errors"
)

var _cmdLong = `
Run tasks across projects in your monorepo.
By default, turbo executes tasks in topological order (i.e.
dependencies first) and then caches the results. Re-running commands for
tasks already in the cache will skip re-execution and immediately move
artifacts from the cache into the correct output folders (as if the task
occurred again).
Arguments passed after '--' will be passed through to the named tasks.
`

// ExecuteRun executes the run command
func ExecuteRun(ctx gocontext.Context, helper *cmdutil.Helper, signalWatcher *signals.Watcher, args *turbostate.ParsedArgsFromRust) error {
base, err := helper.GetCmdBase(args)
Expand Down Expand Up @@ -73,7 +61,9 @@ func optsFromArgs(args *turbostate.ParsedArgsFromRust) (*Opts, error) {

opts := getDefaultOptions()
// aliases := make(map[string]string)
scope.OptsFromArgs(&opts.scopeOpts, args)
if err := scope.OptsFromArgs(&opts.scopeOpts, args); err != nil {
return nil, err
}

// Cache flags
opts.clientOpts.Timeout = args.RemoteCacheTimeout
Expand Down Expand Up @@ -358,6 +348,7 @@ func (r *run) run(ctx gocontext.Context, targets []string) error {
startAt,
r.base.UI,
r.base.RepoRoot,
rs.Opts.scopeOpts.PackageInferenceRoot,
r.base.TurboVersion,
r.base.APIClient,
rs.Opts.runOpts,
Expand All @@ -369,6 +360,7 @@ func (r *run) run(ctx gocontext.Context, targets []string) error {
globalHashable.globalCacheKey,
globalHashable.pipeline,
),
rs.Opts.SynthesizeCommand(rs.Targets),
)

// Dry Run
Expand Down
32 changes: 32 additions & 0 deletions cli/internal/run/run_spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package run

import (
"strings"

"github.com/vercel/turbo/cli/internal/cache"
"github.com/vercel/turbo/cli/internal/client"
"github.com/vercel/turbo/cli/internal/runcache"
Expand Down Expand Up @@ -45,6 +47,36 @@ type Opts struct {
scopeOpts scope.Opts
}

// SynthesizeCommand produces a command that produces an equivalent set of packages, tasks,
// and task arguments to what the current set of opts selects.
func (o *Opts) SynthesizeCommand(tasks []string) string {
cmd := "turbo run"
cmd += " " + strings.Join(tasks, " ")
for _, filterPattern := range o.scopeOpts.FilterPatterns {
cmd += " --filter=" + filterPattern
}
for _, filterPattern := range o.scopeOpts.LegacyFilter.AsFilterPatterns() {
cmd += " --filter=" + filterPattern
}
if o.runOpts.Parallel {
cmd += " --parallel"
}
if o.runOpts.ContinueOnError {
cmd += " --continue"
}
if o.runOpts.DryRun {
if o.runOpts.DryRunJSON {
cmd += " --dry=json"
} else {
cmd += " --dry"
}
}
if len(o.runOpts.PassThroughArgs) > 0 {
cmd += " -- " + strings.Join(o.runOpts.PassThroughArgs, " ")
}
return cmd
}

// getDefaultOptions returns the default set of Opts for every run
func getDefaultOptions() *Opts {
return &Opts{
Expand Down
107 changes: 107 additions & 0 deletions cli/internal/run/run_spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package run

import (
"testing"

"github.com/vercel/turbo/cli/internal/scope"
"github.com/vercel/turbo/cli/internal/util"
)

func TestSynthesizeCommand(t *testing.T) {
testCases := []struct {
filterPatterns []string
legacyFilter scope.LegacyFilter
passThroughArgs []string
parallel bool
continueOnError bool
dryRun bool
dryRunJSON bool
tasks []string
expected string
}{
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
expected: "turbo run build --filter=my-app",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
passThroughArgs: []string{"-v", "--foo=bar"},
expected: "turbo run build --filter=my-app -- -v --foo=bar",
},
{
legacyFilter: scope.LegacyFilter{
Entrypoints: []string{"my-app"},
SkipDependents: true,
},
tasks: []string{"build"},
passThroughArgs: []string{"-v", "--foo=bar"},
expected: "turbo run build --filter=my-app -- -v --foo=bar",
},
{
legacyFilter: scope.LegacyFilter{
Entrypoints: []string{"my-app"},
SkipDependents: true,
},
filterPatterns: []string{"other-app"},
tasks: []string{"build"},
passThroughArgs: []string{"-v", "--foo=bar"},
expected: "turbo run build --filter=other-app --filter=my-app -- -v --foo=bar",
},
{
legacyFilter: scope.LegacyFilter{
Entrypoints: []string{"my-app"},
IncludeDependencies: true,
Since: "some-ref",
},
filterPatterns: []string{"other-app"},
tasks: []string{"build"},
expected: "turbo run build --filter=other-app --filter=...my-app...[some-ref]...",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
parallel: true,
continueOnError: true,
expected: "turbo run build --filter=my-app --parallel --continue",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
dryRun: true,
expected: "turbo run build --filter=my-app --dry",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
dryRun: true,
dryRunJSON: true,
expected: "turbo run build --filter=my-app --dry=json",
},
}

for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.expected, func(t *testing.T) {
o := Opts{
scopeOpts: scope.Opts{
FilterPatterns: testCase.filterPatterns,
LegacyFilter: testCase.legacyFilter,
},
runOpts: util.RunOpts{
PassThroughArgs: testCase.passThroughArgs,
Parallel: testCase.parallel,
ContinueOnError: testCase.continueOnError,
DryRun: testCase.dryRun,
DryRunJSON: testCase.dryRunJSON,
},
}
cmd := o.SynthesizeCommand(testCase.tasks)
if cmd != testCase.expected {
t.Errorf("SynthesizeCommand() got %v, want %v", cmd, testCase.expected)
}
})
}

}
33 changes: 21 additions & 12 deletions cli/internal/runsummary/execution_summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/vercel/turbo/cli/internal/chrometracing"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/turbopath"

"github.com/mitchellh/cli"
)
Expand Down Expand Up @@ -112,10 +113,12 @@ type executionSummary struct {
profileFilename string

// These get serialized to JSON
success int // number of tasks that exited successfully (does not include cache hits)
failure int // number of tasks that exited with failure
cached int // number of tasks that had a cache hit
attempted int // number of tasks that started
command string // a synthesized turbo command to produce this invocation
repoPath turbopath.RelativeSystemPath // the (possibly empty) path from the turborepo root to where the command was run
success int // number of tasks that exited successfully (does not include cache hits)
failure int // number of tasks that exited with failure
cached int // number of tasks that had a cache hit
attempted int // number of tasks that started
startedAt time.Time
endedAt time.Time
exitCode int
Expand All @@ -125,14 +128,18 @@ type executionSummary struct {
// We'll use an anonmyous, private struct for this, so it's not confusingly duplicated.
func (es *executionSummary) MarshalJSON() ([]byte, error) {
serializable := struct {
Success int `json:"success"`
Failure int `json:"failed"`
Cached int `json:"cached"`
Attempted int `json:"attempted"`
StartTime int64 `json:"startTime"`
EndTime int64 `json:"endTime"`
ExitCode int `json:"exitCode"`
Command string `json:"command"`
RepoPath string `json:"repoPath"`
Success int `json:"success"`
Failure int `json:"failed"`
Cached int `json:"cached"`
Attempted int `json:"attempted"`
StartTime int64 `json:"startTime"`
EndTime int64 `json:"endTime"`
ExitCode int `json:"exitCode"`
}{
Command: es.command,
RepoPath: es.repoPath.ToString(),
StartTime: es.startedAt.UnixMilli(),
EndTime: es.endedAt.UnixMilli(),
Success: es.success,
Expand All @@ -146,12 +153,14 @@ func (es *executionSummary) MarshalJSON() ([]byte, error) {
}

// newExecutionSummary creates a executionSummary instance to track events in a `turbo run`.`
func newExecutionSummary(start time.Time, tracingProfile string) *executionSummary {
func newExecutionSummary(command string, repoPath turbopath.RelativeSystemPath, start time.Time, tracingProfile string) *executionSummary {
if tracingProfile != "" {
chrometracing.EnableTracing()
}

return &executionSummary{
command: command,
repoPath: repoPath,
success: 0,
failure: 0,
cached: 0,
Expand Down
50 changes: 22 additions & 28 deletions cli/internal/runsummary/run_summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"path/filepath"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -41,14 +40,16 @@ const (
// Meta is a wrapper around the serializable RunSummary, with some extra information
// about the Run and references to other things that we need.
type Meta struct {
RunSummary *RunSummary
ui cli.Ui
repoRoot turbopath.AbsoluteSystemPath // used to write run summary
singlePackage bool
shouldSave bool
apiClient *client.APIClient
spaceID string
runType runType
RunSummary *RunSummary
ui cli.Ui
repoRoot turbopath.AbsoluteSystemPath // used to write run summary
repoPath turbopath.RelativeSystemPath
singlePackage bool
shouldSave bool
apiClient *client.APIClient
spaceID string
runType runType
synthesizedCommand string
}

// RunSummary contains a summary of what happens in the `turbo run` command and why.
Expand All @@ -67,11 +68,13 @@ func NewRunSummary(
startAt time.Time,
ui cli.Ui,
repoRoot turbopath.AbsoluteSystemPath,
repoPath turbopath.RelativeSystemPath,
turboVersion string,
apiClient *client.APIClient,
runOpts util.RunOpts,
packages []string,
globalHashSummary *GlobalHashSummary,
synthesizedCommand string,
) Meta {
singlePackage := runOpts.SinglePackage
profile := runOpts.Profile
Expand All @@ -86,7 +89,7 @@ func NewRunSummary(
}
}

executionSummary := newExecutionSummary(startAt, profile)
executionSummary := newExecutionSummary(synthesizedCommand, repoPath, startAt, profile)

return Meta{
RunSummary: &RunSummary{
Expand All @@ -98,13 +101,14 @@ func NewRunSummary(
Tasks: []*TaskSummary{},
GlobalHashSummary: globalHashSummary,
},
ui: ui,
runType: runType,
repoRoot: repoRoot,
singlePackage: singlePackage,
shouldSave: shouldSave,
apiClient: apiClient,
spaceID: spaceID,
ui: ui,
runType: runType,
repoRoot: repoRoot,
singlePackage: singlePackage,
shouldSave: shouldSave,
apiClient: apiClient,
spaceID: spaceID,
synthesizedCommand: synthesizedCommand,

This comment has been minimized.

Copy link
@gaspar09

gaspar09 Apr 10, 2023

Contributor

Does repoPath need to be passed here?

This comment has been minimized.

Copy link
@mehulkar

mehulkar Apr 10, 2023

Contributor

it's in the execution summary, actually not sure why it's in both places. i'll get back to you in a bit

}
}

Expand Down Expand Up @@ -181,16 +185,6 @@ func (summary *RunSummary) TrackTask(taskID string) (func(outcome executionEvent
return summary.ExecutionSummary.run(taskID)
}

// command returns a best guess command for the entire Run.
// TODO: we should thread this through from the entry point rather than make it up
func (summary *RunSummary) command() string {
taskNames := make(util.Set, len(summary.Tasks))
for _, task := range summary.Tasks {
taskNames.Add(task.Task)
}
return fmt.Sprintf("turbo run %s", strings.Join(taskNames.UnsafeListOfStrings(), " "))
}

// Save saves the run summary to a file
func (rsm *Meta) save() error {
json, err := rsm.FormatJSON()
Expand Down Expand Up @@ -219,7 +213,7 @@ func (rsm *Meta) record() []error {
// can happen when the Run actually starts, so we can send updates to Vercel as the tasks progress.
runsURL := fmt.Sprintf(runsEndpoint, rsm.spaceID)
var runID string
payload := newVercelRunCreatePayload(rsm.RunSummary)
payload := rsm.newVercelRunCreatePayload()
if startPayload, err := json.Marshal(payload); err == nil {
if resp, err := rsm.apiClient.JSONPost(runsURL, startPayload); err != nil {
errs = append(errs, err)
Expand Down
Loading

0 comments on commit 99a8b68

Please sign in to comment.