chore: Refactor benchmarks

Cleanup benchmarks so that each of them runs on their own mailboxes and
performs the required cleanup after wards.

Benchmarks are now registered via the `RegisterBenchmark()` function and
all existing benchmarks register themselves via `init()`.

The benchmark executable code has been moved to `benchmark.RunMain()` so
that it can be embedded in other projects which implement different
connectors.

Finally a README.md has been added.

benchmarks/gluon_bench/README.md

@@ -0,0 +1,38 @@
+# Gluon Bench - IMAP benchmarks
+
+Gluon bench provides a collection of benchmarks that operate either at the IMAP client level or directly on Gluon 
+itself (e.g: sync).
+
+All IMAP command related benchmarks can be run against a local gluon server which will be started with the benchmark or 
+an externally running IMAP server.
+
+If running against a local server, it's possible to record the execution times of every individual command. 
+
+Finally, it is also possible to produce a JSON report rather than printing to the console.
+
+
+## Building
+
+```bash
+# In benchmarks/gluon_bench
+go build main.go -o gluon_bench 
+```
+
+## Running Gluon Bench
+
+To run Gluon Bench specify a set of options followed by a set of benchmarks you wish to run:
+
+```bash
+gluon_bench -verbose -parallel-client=4 fetch append
+```
+
+Please consult the output of `gluon_bench  -h` for all available options/modifiers and benchmarks.
+
+
+## Integrating Gluon Bench in other projects
+
+When integrating Gluon Bench in other projects which may contain other gluon connectors:
+
+* Register your connector with `utils.RegisterConnector()`
+* Specify the connector with the option `-connector=<...>`
+* In your `main` call `benchmark.RunMain()`

benchmarks/gluon_bench/benchmark/benchmark.go

@@ -19,3 +19,17 @@ type Benchmark interface {
 	// TearDown clear the benchmark state, this is not timed.
 	TearDown(ctx context.Context) error
 }
+
+var benchmarks = make(map[string]Benchmark)
+
+func RegisterBenchmark(benchmark Benchmark) {
+	if _, ok := benchmarks[benchmark.Name()]; ok {
+		panic("Benchmark with this name already exists")
+	}
+
+	benchmarks[benchmark.Name()] = benchmark
+}
+
+func GetBenchmarks() map[string]Benchmark {
+	return benchmarks
+}

benchmarks/gluon_bench/benchmark/run.go

@@ -0,0 +1,144 @@
+package benchmark
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/reporter"
+	"golang.org/x/exp/slices"
+)
+
+func RunMain() {
+	flag.Usage = func() {
+		fmt.Printf("Usage %v [options] benchmark0 benchmark1 ... benchmarkN\n", os.Args[0])
+		fmt.Printf("\nAvailable Benchmarks:\n")
+
+		var benchmarks []string
+
+		for k := range GetBenchmarks() {
+			benchmarks = append(benchmarks, k)
+		}
+
+		slices.Sort(benchmarks)
+
+		for _, k := range benchmarks {
+			fmt.Printf("  * %v\n", k)
+		}
+
+		fmt.Printf("\nOptions:\n")
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+
+	var benchmarks []Benchmark
+
+	args := flag.Args()
+	if len(args) == 0 {
+		flag.Usage()
+		return
+	}
+
+	for _, arg := range args {
+		if v, ok := GetBenchmarks()[arg]; ok {
+			benchmarks = append(benchmarks, v)
+		}
+	}
+
+	if len(benchmarks) == 0 {
+		panic("No benchmarks selected")
+	}
+
+	var benchmarkReporter reporter.BenchmarkReporter
+
+	if len(*flags.JsonReporter) != 0 {
+		benchmarkReporter = reporter.NewJSONReporter(*flags.JsonReporter)
+	} else {
+		benchmarkReporter = &reporter.StdOutReporter{}
+	}
+
+	var benchDirConfig BenchDirConfig
+	if len(*flags.BenchPath) != 0 {
+		benchDirConfig = NewFixedBenchDirConfig(*flags.BenchPath)
+	} else {
+		benchDirConfig = &TmpBenchDirConfig{}
+	}
+
+	benchmarkReports := make([]*reporter.BenchmarkReport, 0, len(benchmarks))
+
+	for _, v := range benchmarks {
+		if *flags.Verbose {
+			fmt.Printf("Begin IMAPBenchmark: %v\n", v.Name())
+		}
+
+		numRuns := *flags.BenchmarkRuns
+
+		var benchmarkRuns = make([]*reporter.BenchmarkRun, 0, numRuns)
+
+		for r := uint(0); r < numRuns; r++ {
+			if *flags.Verbose {
+				fmt.Printf("IMAPBenchmark Run: %v\n", r)
+			}
+
+			benchRun := measureBenchmark(benchDirConfig, r, v)
+			benchmarkRuns = append(benchmarkRuns, benchRun)
+		}
+
+		benchmarkReports = append(benchmarkReports, reporter.NewBenchmarkReport(v.Name(), benchmarkRuns...))
+
+		if *flags.Verbose {
+			fmt.Printf("End IMAPBenchmark: %v\n", v.Name())
+		}
+	}
+
+	if benchmarkReporter != nil {
+		if *flags.Verbose {
+			fmt.Printf("Generating Report\n")
+		}
+
+		if err := benchmarkReporter.ProduceReport(benchmarkReports); err != nil {
+			panic(fmt.Sprintf("Failed to produce benchmark report: %v", err))
+		}
+	}
+
+	if *flags.Verbose {
+		fmt.Printf("Finished\n")
+	}
+}
+
+func measureBenchmark(dirConfig BenchDirConfig, iteration uint, bench Benchmark) *reporter.BenchmarkRun {
+	benchPath, err := dirConfig.Get()
+
+	if err != nil {
+		panic(fmt.Sprintf("Failed to get server directory: %v", err))
+	}
+
+	benchPath = filepath.Join(benchPath, fmt.Sprintf("%v-%d", bench.Name(), iteration))
+
+	if *flags.Verbose {
+		fmt.Printf("IMAPBenchmark Data Path: %v\n", benchPath)
+	}
+
+	if err := os.MkdirAll(benchPath, 0o777); err != nil {
+		panic(fmt.Sprintf("Failed to create server directory '%v' : %v", benchPath, err))
+	}
+
+	ctx := context.Background()
+	if err := bench.Setup(ctx, benchPath); err != nil {
+		panic(fmt.Sprintf("Failed to setup benchmark %v: %v", bench.Name(), err))
+	}
+
+	benchRun, benchErr := bench.Run(ctx)
+	if benchErr != nil {
+		panic(fmt.Sprintf("Failed to run benchmark %v: %v", bench.Name(), err))
+	}
+
+	if err := bench.TearDown(ctx); err != nil {
+		panic(fmt.Sprintf("Failed to teardown benchmark %v: %v", bench.Name(), err))
+	}
+
+	return benchRun
+}

benchmarks/gluon_bench/flags/general.go

@@ -7,7 +7,6 @@ var (
 	Verbose       = flag.Bool("verbose", false, "Enable verbose logging.")
 	JsonReporter  = flag.String("json-reporter", "", "If specified, will generate a json report with the given filename.")
 	BenchmarkRuns = flag.Uint("bench-runs", 1, "Number of runs per benchmark.")
-	ReuseState    = flag.Bool("reuse-state", false, "When present, benchmarks will re-use previous run state, rather than a clean slate.")
 	Connector     = flag.String("connector", "dummy", "Key of the connector implementation registered with ConnectorFactory.")
 	UserName      = flag.String("user-name", "user", "Username for the connector user, defaults to 'user'.")
 	UserPassword  = flag.String("user-pwd", "password", "Password for the connector user, defaults to 'password'.")

benchmarks/gluon_bench/flags/imap_benchmarks.go

@@ -4,8 +4,7 @@ import "flag"
 
 var (
 	RemoteServer          = flag.String("remote-server", "", "IP address and port of the remote IMAP server to run against. E.g. 127.0.0.1:1143.")
-	Mailbox               = flag.String("mailbox", "INBOX", "If not specified will use INBOX as the mailbox to run benchmarks against.")
-	FillSourceMailbox     = flag.Uint("fill-src-mailbox", 1000, "Number of messages to add to the source inbox before each benchmark, set to 0 to skip.")
+	MessageCount          = flag.Uint("msg-count", 1000, "Number of messages to add to the mailbox before each benchmark")
 	RandomSeqSetIntervals = flag.Bool("random-seqset-intervals", false, "When set, generate random sequence intervals rather than single numbers.")
 	UIDMode               = flag.Bool("uid-mode", false, "When set, will run benchmarks in UID mode if available.")
 	ParallelClients       = flag.Uint("parallel-clients", 1, "Set the number of clients to be run in parallel during the benchmark.")

benchmarks/gluon_bench/gluon_benchmarks/sync.go

@@ -165,3 +165,7 @@ func (s *Sync) TearDown(ctx context.Context) error {
 
 	return nil
 }
+
+func init() {
+	benchmark.RegisterBenchmark(NewSync())
+}

benchmarks/gluon_bench/imap_benchmarks/append.go

@@ -0,0 +1,57 @@
+package imap_benchmarks
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/benchmark"
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
+	"github.com/emersion/go-imap/client"
+)
+
+type Append struct {
+	*stateTracker
+}
+
+func NewAppend() benchmark.Benchmark {
+	return NewIMAPBenchmarkRunner(&Append{stateTracker: newStateTracker()})
+}
+
+func (a *Append) Name() string {
+	return "append"
+}
+
+func (a *Append) Setup(ctx context.Context, addr net.Addr) error {
+	if *flags.MessageCount == 0 {
+		return fmt.Errorf("invalid message count")
+	}
+
+	return WithClient(addr, func(cl *client.Client) error {
+		for i := uint(0); i < *flags.ParallelClients; i++ {
+			if _, err := a.createRandomMBox(cl); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func (a *Append) TearDown(ctx context.Context, addr net.Addr) error {
+	return a.cleanupWithAddr(addr)
+}
+
+func (a *Append) Run(ctx context.Context, addr net.Addr) error {
+	RunParallelClients(addr, func(c *client.Client, u uint) {
+		if err := BuildMailbox(c, a.MBoxes[u], int(*flags.MessageCount)); err != nil {
+			panic(err)
+		}
+	})
+
+	return nil
+}
+
+func init() {
+	benchmark.RegisterBenchmark(NewAppend())
+}