cli: add -require-benchmarks (#63)

also fixes #62

.github/workflows/demo.yml

@@ -14,6 +14,7 @@ jobs:
         CHECKS: true
         CHECKS_CONFIG: ./demo/gobenchdata-checks.yml
         PUBLISH_BRANCH: gh-pages
+        GOBENCHDATA_PARSE_FLAGS: --require-benchmarks
       env:
         GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
 
@@ -30,5 +31,6 @@ jobs:
         GO_TEST_FLAGS: -cpu 1,2
         PUBLISH: true
         PUBLISH_BRANCH: gh-pages
+        GOBENCHDATA_PARSE_FLAGS: --require-benchmarks
       env:
         GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}

Dockerfile

@@ -18,4 +18,5 @@ RUN go build -ldflags "-X main.Version=$(git describe --tags)" -o /bin/gobenchda
 RUN rm -rf /tmp/build
 
 # init entrypoint
-ENTRYPOINT ["/bin/gobenchdata", "action"]
+WORKDIR /workdir
+ENTRYPOINT ["/bin/bash", "-c", "export GO_BINARY=/usr/local/go/bin/go && /bin/gobenchdata action"]

action.go

@@ -16,12 +16,13 @@ var entrypointScript string
 func runEmbeddedAction(ctx context.Context) error {
 	cmd := run.Cmd(ctx, "bash").
 		Input(strings.NewReader(entrypointScript)).
+		Environ(os.Environ()).
 		StdOut()
 
 	if executable, err := os.Executable(); err == nil {
 		cmd = cmd.Env(map[string]string{
 			// point to self
-			"GOBENCHDATA": executable,
+			"GOBENCHDATA_BINARY": executable,
 		})
 	}
 

action.yml

@@ -37,6 +37,9 @@ inputs:
     default: 'gh-pages'
     description: branch results are published to
     required: false
+  GOBENCHDATA_PARSE_FLAGS:
+    description: additional flags to pass to gobenchdata when parsing benchmark data
+    required: false
   # publishing
   PUBLISH:
     default: "false"

entrypoint.sh

@@ -2,13 +2,15 @@
 set -e
 
 # core configuration
-export GOBENCHDATA="${GOBENCHDATA:-"gobenchdata"}"
+export GO_BINARY="${GO_BINARY:-"go"}"
+export GOBENCHDATA_BINARY="${GOBENCHDATA_BINARY:-"gobenchdata"}"
 export INPUT_SUBDIRECTORY="${INPUT_SUBDIRECTORY:-"."}"
 export INPUT_PRUNE_COUNT="${INPUT_PRUNE_COUNT:-"0"}"
 export INPUT_BENCHMARKS_OUT="${INPUT_BENCHMARKS_OUT:-"benchmarks.json"}"
 export INPUT_GO_TEST_PKGS="${INPUT_GO_TEST_PKGS:-"./..."}"
 export INPUT_GO_BENCHMARKS="${INPUT_GO_BENCHMARKS:-"."}"
 export INPUT_GIT_COMMIT_MESSAGE="${INPUT_GIT_COMMIT_MESSAGE:-"add benchmark run for ${GITHUB_SHA}"}"
+export INPUT_GOBENCHDATA_PARSE_FLAGS="${INPUT_GOBENCHDATA_PARSE_FLAGS:-""}"
 
 # publishing configuration
 export INPUT_PUBLISH_REPO="${INPUT_PUBLISH_REPO:-${GITHUB_REPOSITORY}}"
@@ -20,9 +22,11 @@ export INPUT_CHECKS_CONFIG="${INPUT_CHECKS_CONFIG:-"gobenchdata-checks.yml"}"
 
 # output build data
 echo '========================'
-command -v ${GOBENCHDATA}
-${GOBENCHDATA} version
 echo "👨‍⚕️ Checking configuration..."
+echo "GO_BINARY=${GO_BINARY}"
+${GO_BINARY} version
+echo "GOBENCHDATA_BINARY=${GOBENCHDATA_BINARY}"
+${GOBENCHDATA_BINARY} version
 env | grep 'INPUT_'
 echo "GITHUB_ACTOR=${GITHUB_ACTOR}"
 echo "GITHUB_WORKSPACE=${GITHUB_WORKSPACE}"
@@ -33,23 +37,19 @@ echo '========================'
 
 # setup
 mkdir -p /tmp/{gobenchdata,build}
-if [[ "${SET_GIT_USER}" != "false" ]]; then
-  git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
-  git config user.name "${GITHUB_ACTOR}"
-fi
 
 # run benchmarks from configured directory
 echo
 echo '📊 Running benchmarks...'
 RUN_OUTPUT="/tmp/gobenchdata/benchmarks.json"
 cd "${GITHUB_WORKSPACE}"
 cd "${INPUT_SUBDIRECTORY}"
-go test \
+${GO_BINARY} test \
   -bench "${INPUT_GO_BENCHMARKS}" \
   -benchmem \
   ${INPUT_GO_TEST_FLAGS} \
   ${INPUT_GO_TEST_PKGS} |
-  ${GOBENCHDATA} --json "${RUN_OUTPUT}" -v "${GITHUB_SHA}" -t "ref=${GITHUB_REF}"
+  ${GOBENCHDATA_BINARY} ${INPUT_GOBENCHDATA_PARSE_FLAGS} --json "${RUN_OUTPUT}" -v "${GITHUB_SHA}" -t "ref=${GITHUB_REF}"
 cd "${GITHUB_WORKSPACE}"
 
 # fetch published data
@@ -66,7 +66,7 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then
     # check results against published
     echo '🔎 Evaluating results against base runs...'
     CHECKS_OUTPUT="/tmp/gobenchdata/checks-results.json"
-    ${GOBENCHDATA} checks eval "${INPUT_BENCHMARKS_OUT}" "${RUN_OUTPUT}" \
+    ${GOBENCHDATA_BINARY} checks eval "${INPUT_BENCHMARKS_OUT}" "${RUN_OUTPUT}" \
       --checks.config "${GITHUB_WORKSPACE}/${INPUT_CHECKS_CONFIG}" \
       --json ${CHECKS_OUTPUT} \
       --flat
@@ -76,7 +76,7 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then
     # output results
     echo
     echo '📝 Generating checks report...'
-    ${GOBENCHDATA} checks report ${CHECKS_OUTPUT}
+    ${GOBENCHDATA_BINARY} checks report ${CHECKS_OUTPUT}
 
   fi
 
@@ -86,7 +86,7 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then
     echo '☝️ Updating results...'
     if [[ -f "${INPUT_BENCHMARKS_OUT}" ]]; then
       echo '📈 Existing report found - merging...'
-      ${GOBENCHDATA} merge "${RUN_OUTPUT}" "${INPUT_BENCHMARKS_OUT}" \
+      ${GOBENCHDATA_BINARY} merge "${RUN_OUTPUT}" "${INPUT_BENCHMARKS_OUT}" \
         --prune "${INPUT_PRUNE_COUNT}" \
         --json "${INPUT_BENCHMARKS_OUT}" \
         --flat
@@ -97,6 +97,10 @@ if [[ "${INPUT_PUBLISH}" == "true" || "${INPUT_CHECKS}" == "true" ]]; then
     # publish results
     echo
     echo '📷 Committing and pushing new benchmark data...'
+    if [[ "${SET_GIT_USER}" != "false" ]]; then
+      git config --local user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+      git config --local user.name "${GITHUB_ACTOR}"
+    fi
     git add .
     git commit -m "${INPUT_GIT_COMMIT_MESSAGE}"
     git push -f origin ${INPUT_PUBLISH_BRANCH}

main.go

@@ -28,6 +28,8 @@ var (
 	noSort    = pflag.Bool("no-sort", false, "disable sorting")
 	prune     = pflag.Int("prune", 0, "number of runs to keep (default: keep all)")
 
+	requireBenchmarks = pflag.Bool("require-benchmarks", false, "fail if no benchmarks detected")
+
 	webConfigOnly = pflag.Bool("web.config-only", false, "only generate configuration for 'gobenchdata web'")
 	webIndexTitle = pflag.String("web.title", "gobenchdata web", "header <title> for 'gobenchdata web'")
 	webIndexHead  = pflag.StringArray("web.head", []string{}, "additional <head> elements for 'gobenchdata web'")
@@ -112,7 +114,7 @@ func main() {
 
 				if !*webConfigOnly {
 					if err := web.GenerateApp(dir, *it); err != nil {
-						fmt.Println(err.Error())
+						println(err.Error())
 						os.Exit(1)
 					}
 					fmt.Println("web application generated!")
@@ -123,7 +125,7 @@ func main() {
 						fmt.Println("found existing web app configuration")
 						return
 					}
-					fmt.Println(err.Error())
+					println(err.Error())
 					os.Exit(1)
 				}
 				fmt.Println("web application configuration generated!")
@@ -138,7 +140,7 @@ func main() {
 				fmt.Printf("serving './benchmarks.json' on '%s'\n", addr)
 				go internal.OpenBrowser("http://" + addr)
 				if err := web.ListenAndServe(addr, *config, *it); err != nil {
-					fmt.Println(err.Error())
+					println(err.Error())
 					os.Exit(1)
 				}
 			default:
@@ -155,13 +157,13 @@ func main() {
 			switch checksCmd := pflag.Args()[1]; checksCmd {
 			case "generate":
 				if err := checks.GenerateConfig(*checksConfigPath); err != nil {
-					fmt.Println(err.Error())
+					println(err.Error())
 					os.Exit(1)
 				}
 			case "eval":
 				cfg, err := checks.LoadConfig(*checksConfigPath)
 				if err != nil {
-					fmt.Println(err.Error())
+					println(err.Error())
 					os.Exit(1)
 				}
 				args := pflag.Args()[2:]
@@ -175,7 +177,7 @@ func main() {
 					MustFindAll: false,
 				})
 				if err != nil {
-					fmt.Println(err.Error())
+					println(err.Error())
 					os.Exit(1)
 				}
 
@@ -190,12 +192,12 @@ func main() {
 						b, err = json.MarshalIndent(results, "", "  ")
 					}
 					if err != nil {
-						fmt.Println(err.Error())
+						println(err.Error())
 						os.Exit(1)
 					}
 
 					if err := ioutil.WriteFile(*jsonOut, b, os.ModePerm); err != nil {
-						fmt.Println(err.Error())
+						println(err.Error())
 						os.Exit(1)
 					}
 					fmt.Printf("report output written to %s\n", *jsonOut)
@@ -208,7 +210,7 @@ func main() {
 				}
 				results, err := checks.LoadReport(pflag.Args()[2])
 				if err != nil {
-					fmt.Println(err.Error())
+					println(err.Error())
 					os.Exit(1)
 				}
 				outputChecksReport(results)
@@ -225,7 +227,7 @@ func main() {
 
 		case "action":
 			if err := runEmbeddedAction(context.Background()); err != nil {
-				fmt.Println(err.Error())
+				println(err.Error())
 				os.Exit(1)
 			}
 
@@ -239,7 +241,7 @@ func main() {
 	// default behaviour
 	fi, err := os.Stdin.Stat()
 	if err != nil {
-		fmt.Println(err.Error())
+		println(err.Error())
 		os.Exit(1)
 	} else if fi.Mode()&os.ModeNamedPipe == 0 {
 		fmt.Println("gobenchdata should be used with a pipe - see 'gobenchdata help'")
@@ -249,10 +251,14 @@ func main() {
 	parser := bench.NewParser(bufio.NewReader(os.Stdin))
 	suites, err := parser.Read()
 	if err != nil {
-		fmt.Println(err.Error())
+		println(err.Error())
 		os.Exit(1)
 	}
 	fmt.Printf("detected %d benchmark suites\n", len(suites))
+	if *requireBenchmarks && len(suites) == 0 {
+		println("expected benchmarks suites to be detected, found none")
+		os.Exit(1)
+	}
 
 	// set up results
 	results := []bench.Run{{
@@ -268,12 +274,12 @@ func main() {
 		}
 		b, err := ioutil.ReadFile(*jsonOut)
 		if err != nil && !os.IsNotExist(err) {
-			fmt.Println(err.Error())
+			println(err.Error())
 			os.Exit(1)
 		} else if !os.IsNotExist(err) {
 			var runs []bench.Run
 			if err := json.Unmarshal(b, &runs); err != nil {
-				fmt.Println(err.Error())
+				println(err.Error())
 				os.Exit(1)
 			}
 			results = append(results, runs...)