From 87501dc2d1902193b0aee21073e10cc5e6b688de Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Sat, 11 Jun 2022 13:02:28 -0700 Subject: [PATCH] cli: add -require-benchmarks (#63) also fixes #62 --- .github/workflows/demo.yml | 2 ++ Dockerfile | 3 ++- action.go | 3 ++- action.yml | 3 +++ entrypoint.sh | 28 ++++++++++++++++------------ main.go | 34 ++++++++++++++++++++-------------- 6 files changed, 45 insertions(+), 28 deletions(-) diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index 752c997..686ab52 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -14,6 +14,7 @@ jobs: CHECKS: true CHECKS_CONFIG: ./demo/gobenchdata-checks.yml PUBLISH_BRANCH: gh-pages + GOBENCHDATA_PARSE_FLAGS: --require-benchmarks env: GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} @@ -30,5 +31,6 @@ jobs: GO_TEST_FLAGS: -cpu 1,2 PUBLISH: true PUBLISH_BRANCH: gh-pages + GOBENCHDATA_PARSE_FLAGS: --require-benchmarks env: GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} diff --git a/Dockerfile b/Dockerfile index ec9f6f3..5040c89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,5 @@ RUN go build -ldflags "-X main.Version=$(git describe --tags)" -o /bin/gobenchda RUN rm -rf /tmp/build # init entrypoint -ENTRYPOINT ["/bin/gobenchdata", "action"] +WORKDIR /workdir +ENTRYPOINT ["/bin/bash", "-c", "export GO_BINARY=/usr/local/go/bin/go && /bin/gobenchdata action"] diff --git a/action.go b/action.go index c73900e..6af94be 100644 --- a/action.go +++ b/action.go @@ -16,12 +16,13 @@ var entrypointScript string func runEmbeddedAction(ctx context.Context) error { cmd := run.Cmd(ctx, "bash"). Input(strings.NewReader(entrypointScript)). + Environ(os.Environ()). StdOut() if executable, err := os.Executable(); err == nil { cmd = cmd.Env(map[string]string{ // point to self - "GOBENCHDATA": executable, + "GOBENCHDATA_BINARY": executable, }) } diff --git a/action.yml b/action.yml index 3d0a8d6..5a02c6f 100644 --- a/action.yml +++ b/action.yml @@ -37,6 +37,9 @@ inputs: default: 'gh-pages' description: branch results are published to required: false + GOBENCHDATA_PARSE_FLAGS: + description: additional flags to pass to gobenchdata when parsing benchmark data + required: false # publishing PUBLISH: default: "false" diff --git a/entrypoint.sh b/entrypoint.sh index be6f7d7..ca857fc 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,13 +2,15 @@ set -e # core configuration -export GOBENCHDATA="${GOBENCHDATA:-"gobenchdata"}" +export GO_BINARY="${GO_BINARY:-"go"}" +export GOBENCHDATA_BINARY="${GOBENCHDATA_BINARY:-"gobenchdata"}" export INPUT_SUBDIRECTORY="${INPUT_SUBDIRECTORY:-"."}" export INPUT_PRUNE_COUNT="${INPUT_PRUNE_COUNT:-"0"}" export INPUT_BENCHMARKS_OUT="${INPUT_BENCHMARKS_OUT:-"benchmarks.json"}" export INPUT_GO_TEST_PKGS="${INPUT_GO_TEST_PKGS:-"./..."}" export INPUT_GO_BENCHMARKS="${INPUT_GO_BENCHMARKS:-"."}" export INPUT_GIT_COMMIT_MESSAGE="${INPUT_GIT_COMMIT_MESSAGE:-"add benchmark run for ${GITHUB_SHA}"}" +export INPUT_GOBENCHDATA_PARSE_FLAGS="${INPUT_GOBENCHDATA_PARSE_FLAGS:-""}" # publishing configuration export INPUT_PUBLISH_REPO="${INPUT_PUBLISH_REPO:-${GITHUB_REPOSITORY}}" @@ -20,9 +22,11 @@ export INPUT_CHECKS_CONFIG="${INPUT_CHECKS_CONFIG:-"gobenchdata-checks.yml"}" # output build data echo '========================' -command -v ${GOBENCHDATA} -${GOBENCHDATA} version echo "👨‍⚕️ Checking configuration..." +echo "GO_BINARY=${GO_BINARY}" +${GO_BINARY} version +echo "GOBENCHDATA_BINARY=${GOBENCHDATA_BINARY}" +${GOBENCHDATA_BINARY} version env | grep 'INPUT_' echo "GITHUB_ACTOR=${GITHUB_ACTOR}" echo "GITHUB_WORKSPACE=${GITHUB_WORKSPACE}" @@ -33,10 +37,6 @@ echo '========================' # setup mkdir -p /tmp/{gobenchdata,build} -if [[ "${SET_GIT_USER}" != "false" ]]; then - git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" - git config user.name "${GITHUB_ACTOR}" -fi # run benchmarks from configured directory echo @@ -44,12 +44,12 @@ echo '📊 Running benchmarks...' RUN_OUTPUT="/tmp/gobenchdata/benchmarks.json" cd "${GITHUB_WORKSPACE}" cd "${INPUT_SUBDIRECTORY}" -go test \ +${GO_BINARY} test \ -bench "${INPUT_GO_BENCHMARKS}" \ -benchmem \ ${INPUT_GO_TEST_FLAGS} \ ${INPUT_GO_TEST_PKGS} | - ${GOBENCHDATA} --json "${RUN_OUTPUT}" -v "${GITHUB_SHA}" -t "ref=${GITHUB_REF}" + ${GOBENCHDATA_BINARY} ${INPUT_GOBENCHDATA_PARSE_FLAGS} --json "${RUN_OUTPUT}" -v "${GITHUB_SHA}" -t "ref=${GITHUB_REF}" cd "${GITHUB_WORKSPACE}" # fetch published data @@ -66,7 +66,7 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then # check results against published echo '🔎 Evaluating results against base runs...' CHECKS_OUTPUT="/tmp/gobenchdata/checks-results.json" - ${GOBENCHDATA} checks eval "${INPUT_BENCHMARKS_OUT}" "${RUN_OUTPUT}" \ + ${GOBENCHDATA_BINARY} checks eval "${INPUT_BENCHMARKS_OUT}" "${RUN_OUTPUT}" \ --checks.config "${GITHUB_WORKSPACE}/${INPUT_CHECKS_CONFIG}" \ --json ${CHECKS_OUTPUT} \ --flat @@ -76,7 +76,7 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then # output results echo echo '📝 Generating checks report...' - ${GOBENCHDATA} checks report ${CHECKS_OUTPUT} + ${GOBENCHDATA_BINARY} checks report ${CHECKS_OUTPUT} fi @@ -86,7 +86,7 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then echo '☝️ Updating results...' if [[ -f "${INPUT_BENCHMARKS_OUT}" ]]; then echo '📈 Existing report found - merging...' - ${GOBENCHDATA} merge "${RUN_OUTPUT}" "${INPUT_BENCHMARKS_OUT}" \ + ${GOBENCHDATA_BINARY} merge "${RUN_OUTPUT}" "${INPUT_BENCHMARKS_OUT}" \ --prune "${INPUT_PRUNE_COUNT}" \ --json "${INPUT_BENCHMARKS_OUT}" \ --flat @@ -97,6 +97,10 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then # publish results echo echo '📷 Committing and pushing new benchmark data...' + if [[ "${SET_GIT_USER}" != "false" ]]; then + git config --local user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git config --local user.name "${GITHUB_ACTOR}" + fi git add . git commit -m "${INPUT_GIT_COMMIT_MESSAGE}" git push -f origin ${INPUT_PUBLISH_BRANCH} diff --git a/main.go b/main.go index 12d54ca..e999fc6 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,8 @@ var ( noSort = pflag.Bool("no-sort", false, "disable sorting") prune = pflag.Int("prune", 0, "number of runs to keep (default: keep all)") + requireBenchmarks = pflag.Bool("require-benchmarks", false, "fail if no benchmarks detected") + webConfigOnly = pflag.Bool("web.config-only", false, "only generate configuration for 'gobenchdata web'") webIndexTitle = pflag.String("web.title", "gobenchdata web", "header for 'gobenchdata web'") webIndexHead = pflag.StringArray("web.head", []string{}, "additional <head> elements for 'gobenchdata web'") @@ -112,7 +114,7 @@ func main() { if !*webConfigOnly { if err := web.GenerateApp(dir, *it); err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } fmt.Println("web application generated!") @@ -123,7 +125,7 @@ func main() { fmt.Println("found existing web app configuration") return } - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } fmt.Println("web application configuration generated!") @@ -138,7 +140,7 @@ func main() { fmt.Printf("serving './benchmarks.json' on '%s'\n", addr) go internal.OpenBrowser("http://" + addr) if err := web.ListenAndServe(addr, *config, *it); err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } default: @@ -155,13 +157,13 @@ func main() { switch checksCmd := pflag.Args()[1]; checksCmd { case "generate": if err := checks.GenerateConfig(*checksConfigPath); err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } case "eval": cfg, err := checks.LoadConfig(*checksConfigPath) if err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } args := pflag.Args()[2:] @@ -175,7 +177,7 @@ func main() { MustFindAll: false, }) if err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } @@ -190,12 +192,12 @@ func main() { b, err = json.MarshalIndent(results, "", " ") } if err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } if err := ioutil.WriteFile(*jsonOut, b, os.ModePerm); err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } fmt.Printf("report output written to %s\n", *jsonOut) @@ -208,7 +210,7 @@ func main() { } results, err := checks.LoadReport(pflag.Args()[2]) if err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } outputChecksReport(results) @@ -225,7 +227,7 @@ func main() { case "action": if err := runEmbeddedAction(context.Background()); err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } @@ -239,7 +241,7 @@ func main() { // default behaviour fi, err := os.Stdin.Stat() if err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } else if fi.Mode()&os.ModeNamedPipe == 0 { fmt.Println("gobenchdata should be used with a pipe - see 'gobenchdata help'") @@ -249,10 +251,14 @@ func main() { parser := bench.NewParser(bufio.NewReader(os.Stdin)) suites, err := parser.Read() if err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } fmt.Printf("detected %d benchmark suites\n", len(suites)) + if *requireBenchmarks && len(suites) == 0 { + println("expected benchmarks suites to be detected, found none") + os.Exit(1) + } // set up results results := []bench.Run{{ @@ -268,12 +274,12 @@ func main() { } b, err := ioutil.ReadFile(*jsonOut) if err != nil && !os.IsNotExist(err) { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } else if !os.IsNotExist(err) { var runs []bench.Run if err := json.Unmarshal(b, &runs); err != nil { - fmt.Println(err.Error()) + println(err.Error()) os.Exit(1) } results = append(results, runs...)