New benchmark system.

Two files that do rouglhly the same thing, one written in Primi and one written in PHP.
Primi performance is now evaluated "how many times is Primi slower than an equivalent code in PHP?" This should make benchmarks more relevant - and any performance improvements or regressions should be visible better.

Currently on my machine:
02.03.2020 22:35, 27.73, perf 37.86x slower vs 0.47678733

... this is inclusing changes that are not yet commited.

bin/bench

@@ -5,38 +5,69 @@ cd $(dirname $0)/..
 # Time format for 'time' command.
 TIMEFORMAT=%R
 
-iterations=10
-echo "Iterations: $iterations"
-
 echo "Clearing AST cache ..."
 rm ./temp/ast*.json 2>/dev/null
 rm bench.out 2>/dev/null
 
-simplefiles=$(find ./example -iname "bench*.primi")
-simpletime=0
+ITERS=3
+PERF_STANDARD_PATH='./tests/bench/perf_standard.php'
+BENCH_FILES=$(find ./tests/bench/ -iname "bench*.primi")
+SIMPLE_TIME=0
+TOTAL_TIME=0
+AVG_SCORE=0
+
+function get_precise_time {
+    date +%s.%N
+}
+
+function measure_time {
+    # We probably should measure only user+kernel time our process really took
+    # (without I/O waits and stuff), but /usr/bin/time, which is handy, returns
+    # insufficient precision (only 2 decimal points).
+    # So lets measure everything.
+    START=`get_precise_time`
+    TMP=`$1 > /dev/null`
+    END=`get_precise_time`
+    perl -e "printf('%.8f', $END - $START);"
+}
+
+function timeit_php {
+    echo `measure_time "php $1"`
+}
 
-function timeit {
-    echo `{ time ./primi $1 >> bench.out; } 2>&1`
+function timeit_primi {
+    echo `measure_time "./primi $1"`
 }
 
-echo "Running benchmark ..."
-for i in $(seq $iterations)
+echo -n "Measuring perf standard ... "
+PERF_STD=`timeit_php $PERF_STANDARD_PATH`
+echo "$PERF_STD s"
+
+echo "Running benchmarks ..."
+for i in $(seq $ITERS)
 do
-    echo "█ ($i / $iterations) "
-    for f in $simplefiles
+    [[ "$i" == "1" ]] && STATE='parsing' || STATE='cached'
+
+    [[ "$STATE" == "parsing" ]] && DESC="With AST parsing" || DESC="With cached AST"
+    echo "█ $DESC ($i / $ITERS)"
+
+    for f in $BENCH_FILES
     do
         echo -n "$f ... "
-        tmp=$(timeit $f)
-        echo "$tmp s";
-        simpletime=$(perl -e "printf('%.2f', $simpletime + $tmp);")
+        SIMPLE_TIME=$(timeit_primi $f)
+        SCORE=$(perl -e "printf('%.2f', $SIMPLE_TIME / $PERF_STD)")
+        TOTAL_SCORE=$(perl -e "printf('%.2f', $SCORE + $TOTAL_TIME)")
+        TOTAL_TIME=$(perl -e "printf('%.2f', $SIMPLE_TIME + $TOTAL_TIME)")
+        echo "$SIMPLE_TIME s (${SCORE}x slower)";
     done
 done
 
+AVG_TIME=`perl -e "printf('%.2f', $TOTAL_TIME / $ITERS);"`
+AVG_SCORE=`perl -e "printf('%.2f', $TOTAL_SCORE / $ITERS);"`
 printf \
 "Results:\n"\
-"- Total: $simpletime"" s \n"\
-"- AVG  : "`perl -e "printf('%.2f', $simpletime / $iterations);"`" s \n"
+"- Total: $TOTAL_TIME s\n"\
+"- AVG  : $AVG_TIME s\n"
 
-nowdate=`date +"%d.%m.%Y %H:%M"`
-average=`perl -e "printf('%.2f', $simpletime / $iterations);"`
-echo "$nowdate,$iterations,$simpletime,$average" >> "bench_progress.csv"
+TODAY=`date +"%d.%m.%Y %H:%M"`
+echo "$TODAY, ${AVG_TIME//,/.}, perf ${AVG_SCORE//,/.}x slower vs ${PERF_STD//,/.}" >> "bench_progress.csv"

tests/bench/bench_perf.primi

@@ -0,0 +1,38 @@
+max_iter = 500000
+
+function bench_function_calls() {
+
+	adder = function(x, y) {
+		return x + y
+	}
+
+	result = -1024
+	c = 0
+	while (c < max_iter) {
+		result = result + adder(c, 1)
+		c = c + 1
+	}
+
+	return c
+
+}
+
+function bench_regex_matches() {
+
+	haystack = "Když začínáme myslet, nemáme k dispozici nic jiného než myšlenku v " +
+		"její čisté neurčenosti, neboť k určení již patří jedno nebo nějaké " +
+		"jiné, ale na začátku ještě nemáme žádné jiné..."
+
+	result = 0
+	c = 0
+	while (c < max_iter) {
+		result = result + (haystack == r"^.*(zač).*(,)?.*?(\.)").to_number()
+		c = c + 1
+	}
+
+	return c
+
+}
+
+bench_function_calls()
+bench_regex_matches()

tests/bench/standard_perf.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+ini_set('display_errors', '1');
+ini_set('display_startup_errors', '1');
+error_reporting(E_ALL);
+const MAX_ITER = 500000;
+
+//
+// Run some CPU intensive tasks and measure their duration in pure PHP.
+//
+
+function measure($fn) {
+	$start = microtime(true);
+	$fn();
+	return microtime(true) - $start;
+}
+
+function bench_function_calls() {
+
+	$adder = function($x, $y) {
+		return $x + $y;
+	};
+
+	$result = -1024;
+	$c = 0;
+	while ($c < MAX_ITER) {
+		$result = $result + $adder($c, 1);
+		$c = $c + 1;
+	}
+
+	return $c;
+
+}
+
+function bench_regex_matches() {
+
+	$haystack = "Když začínáme myslet, nemáme k dispozici nic jiného než myšlenku v " .
+				"její čisté neurčenosti, neboť k určení již patří jedno nebo nějaké " .
+				"jiné, ale na začátku ještě nemáme žádné jiné...";
+
+	$result = 0;
+	$c = 0;
+	while ($c < MAX_ITER) {
+		$result += preg_match('#^.*(zač).*(,)?.*?(\.)$#', $haystack, $m);
+		$c = $c + 1;
+	}
+
+	return $c;
+
+}
+
+$results = [
+	'bench_function_calls' => measure('bench_function_calls'),
+	'bench_regex_matches' => measure('bench_regex_matches'),
+];
+
+//
+// Print results in INI format for easy parsing, if needed.
+//
+
+$total = 0;
+foreach ($results as $name => $time) {
+	echo "$name=$time\n";
+	$total += $time;
+}
+echo "total=$total\n";