WIP: Add schnorrsig batch verification

doc/speedup-batch.md

@@ -0,0 +1,4 @@
+# Schnorrsig Batch Verification Speedup
+
+![Speedup over single verification](speedup-batch/speedup-batch.png)
+

doc/speedup-batch/.gitignore

@@ -0,0 +1,2 @@
+batch.dat
+single.dat

doc/speedup-batch/Makefile

@@ -0,0 +1,11 @@
+bench_output.txt: bench.sh
+	SECP256K1_BENCH_ITERS=500000 ./bench.sh bench_output.txt
+
+batch.dat: bench_output.txt
+	cat bench_output.txt | grep -v "schnorrsig_batch_verify_1:" | gawk 'match($$0, /schnorrsig_batch_verify_(.*):.*avg (.*)us /, a) {print a[1] " " a[2]}' > batch.dat
+
+single.dat: bench_output.txt
+	cat bench_output.txt | awk 'match($$0, /schnorrsig_verify:.*avg (.*)us /, a) {print a[1]}' > single.dat
+
+speedup-batch.png: batch.dat single.dat plot.p
+	gnuplot plot.p

doc/speedup-batch/bench.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+output_file=$1
+cur_dir=$(pwd)
+
+cd ../../
+echo "HEAD: $(git rev-parse --short HEAD)" > "$cur_dir/$output_file.log"
+make clean
+./autogen.sh
+./configure --enable-experimental --enable-module-schnorrsig >> "$cur_dir/$output_file.log"
+make -j
+./bench_schnorrsig > "$cur_dir/$output_file"
+

doc/speedup-batch/bench_output.txt

@@ -0,0 +1,67 @@
+schnorrsig_sign: min 24.3us / avg 24.3us / max 24.4us
+schnorrsig_verify: min 41.9us / avg 42.0us / max 42.0us
+schnorrsig_batch_verify_1: min 50.0us / avg 50.1us / max 50.1us
+schnorrsig_batch_verify_2: min 42.1us / avg 42.1us / max 42.1us
+schnorrsig_batch_verify_3: min 39.3us / avg 39.3us / max 39.4us
+schnorrsig_batch_verify_4: min 38.0us / avg 38.0us / max 38.1us
+schnorrsig_batch_verify_5: min 37.2us / avg 37.2us / max 37.2us
+schnorrsig_batch_verify_7: min 36.2us / avg 36.2us / max 36.3us
+schnorrsig_batch_verify_9: min 35.6us / avg 35.7us / max 35.7us
+schnorrsig_batch_verify_11: min 35.3us / avg 35.4us / max 35.4us
+schnorrsig_batch_verify_14: min 35.0us / avg 35.0us / max 35.0us
+schnorrsig_batch_verify_17: min 34.7us / avg 34.7us / max 34.8us
+schnorrsig_batch_verify_21: min 34.5us / avg 34.6us / max 34.6us
+schnorrsig_batch_verify_26: min 34.4us / avg 34.4us / max 34.4us
+schnorrsig_batch_verify_32: min 34.3us / avg 34.3us / max 34.3us
+schnorrsig_batch_verify_39: min 34.2us / avg 34.2us / max 34.2us
+schnorrsig_batch_verify_47: min 33.1us / avg 33.1us / max 33.2us
+schnorrsig_batch_verify_57: min 32.1us / avg 32.1us / max 32.1us
+schnorrsig_batch_verify_69: min 32.0us / avg 32.0us / max 32.0us
+schnorrsig_batch_verify_83: min 30.8us / avg 30.8us / max 30.8us
+schnorrsig_batch_verify_100: min 29.8us / avg 29.8us / max 29.8us
+schnorrsig_batch_verify_121: min 30.0us / avg 30.0us / max 30.0us
+schnorrsig_batch_verify_146: min 28.8us / avg 28.8us / max 28.9us
+schnorrsig_batch_verify_176: min 27.9us / avg 27.9us / max 27.9us
+schnorrsig_batch_verify_212: min 27.1us / avg 27.1us / max 27.1us
+schnorrsig_batch_verify_255: min 26.4us / avg 26.4us / max 26.5us
+schnorrsig_batch_verify_307: min 25.8us / avg 25.8us / max 25.9us
+schnorrsig_batch_verify_369: min 25.4us / avg 25.4us / max 25.4us
+schnorrsig_batch_verify_443: min 25.0us / avg 25.0us / max 25.0us
+schnorrsig_batch_verify_532: min 24.7us / avg 24.7us / max 24.8us
+schnorrsig_batch_verify_639: min 25.2us / avg 25.2us / max 25.2us
+schnorrsig_batch_verify_767: min 24.5us / avg 24.5us / max 24.5us
+schnorrsig_batch_verify_921: min 23.9us / avg 23.9us / max 23.9us
+schnorrsig_batch_verify_1106: min 23.4us / avg 23.4us / max 23.4us
+schnorrsig_batch_verify_1328: min 23.0us / avg 23.1us / max 23.1us
+schnorrsig_batch_verify_1594: min 22.7us / avg 22.7us / max 22.7us
+schnorrsig_batch_verify_1913: min 22.3us / avg 22.4us / max 22.4us
+schnorrsig_batch_verify_2296: min 22.4us / avg 22.4us / max 22.5us
+schnorrsig_batch_verify_2756: min 22.1us / avg 22.1us / max 22.1us
+schnorrsig_batch_verify_3308: min 21.8us / avg 21.8us / max 21.8us
+schnorrsig_batch_verify_3970: min 21.9us / avg 21.9us / max 21.9us
+schnorrsig_batch_verify_4765: min 21.5us / avg 21.6us / max 21.6us
+schnorrsig_batch_verify_5719: min 21.2us / avg 21.2us / max 21.2us
+schnorrsig_batch_verify_6863: min 21.0us / avg 21.0us / max 21.0us
+schnorrsig_batch_verify_8236: min 21.0us / avg 21.0us / max 21.0us
+schnorrsig_batch_verify_9884: min 20.7us / avg 20.7us / max 20.7us
+schnorrsig_batch_verify_11861: min 20.5us / avg 20.5us / max 20.5us
+schnorrsig_batch_verify_14234: min 20.2us / avg 20.3us / max 20.3us
+schnorrsig_batch_verify_17081: min 20.1us / avg 20.1us / max 20.1us
+schnorrsig_batch_verify_20498: min 20.0us / avg 20.0us / max 20.0us
+schnorrsig_batch_verify_24598: min 19.8us / avg 19.8us / max 19.8us
+schnorrsig_batch_verify_29518: min 19.7us / avg 19.7us / max 19.7us
+schnorrsig_batch_verify_35422: min 19.6us / avg 19.6us / max 19.6us
+schnorrsig_batch_verify_42507: min 19.6us / avg 19.6us / max 19.6us
+schnorrsig_batch_verify_51009: min 19.5us / avg 19.5us / max 19.6us
+schnorrsig_batch_verify_61211: min 19.5us / avg 19.5us / max 19.5us
+schnorrsig_batch_verify_73454: min 19.4us / avg 19.4us / max 19.4us
+schnorrsig_batch_verify_88145: min 19.4us / avg 19.5us / max 19.5us
+schnorrsig_batch_verify_105775: min 19.4us / avg 19.4us / max 19.4us
+schnorrsig_batch_verify_126931: min 19.3us / avg 19.4us / max 19.4us
+schnorrsig_batch_verify_152318: min 19.3us / avg 19.3us / max 19.3us
+schnorrsig_batch_verify_182782: min 19.3us / avg 19.3us / max 19.3us
+schnorrsig_batch_verify_219339: min 19.3us / avg 19.4us / max 19.4us
+schnorrsig_batch_verify_263207: min 19.3us / avg 19.4us / max 19.4us
+schnorrsig_batch_verify_315849: min 19.3us / avg 19.3us / max 19.4us
+schnorrsig_batch_verify_379019: min 19.3us / avg 19.4us / max 19.4us
+schnorrsig_batch_verify_454823: min 19.3us / avg 19.3us / max 19.4us

doc/speedup-batch/bench_output.txt.log

@@ -0,0 +1,129 @@
+HEAD: 2d843581
+checking build system type... x86_64-pc-linux-gnu
+checking host system type... x86_64-pc-linux-gnu
+checking for a BSD-compatible install... /usr/bin/install -c
+checking whether build environment is sane... yes
+checking for a race-free mkdir -p... /usr/bin/mkdir -p
+checking for gawk... gawk
+checking whether make sets $(MAKE)... yes
+checking whether make supports nested variables... yes
+checking how to print strings... printf
+checking whether make supports the include directive... yes (GNU style)
+checking for gcc... gcc
+checking whether the C compiler works... yes
+checking for C compiler default output file name... a.out
+checking for suffix of executables... 
+checking whether we are cross compiling... no
+checking for suffix of object files... o
+checking whether the compiler supports GNU C... yes
+checking whether gcc accepts -g... yes
+checking for gcc option to enable C11 features... none needed
+checking whether gcc understands -c and -o together... yes
+checking dependency style of gcc... gcc3
+checking for a sed that does not truncate output... /usr/bin/sed
+checking for grep that handles long lines and -e... /usr/bin/grep
+checking for egrep... /usr/bin/grep -E
+checking for fgrep... /usr/bin/grep -F
+checking for ld used by gcc... /usr/bin/ld
+checking if the linker (/usr/bin/ld) is GNU ld... yes
+checking for BSD- or MS-compatible name lister (nm)... /usr/bin/nm -B
+checking the name lister (/usr/bin/nm -B) interface... BSD nm
+checking whether ln -s works... yes
+checking the maximum length of command line arguments... 1572864
+checking how to convert x86_64-pc-linux-gnu file names to x86_64-pc-linux-gnu format... func_convert_file_noop
+checking how to convert x86_64-pc-linux-gnu file names to toolchain format... func_convert_file_noop
+checking for /usr/bin/ld option to reload object files... -r
+checking for objdump... objdump
+checking how to recognize dependent libraries... pass_all
+checking for dlltool... no
+checking how to associate runtime and link libraries... printf %s\n
+checking for ar... ar
+checking for archiver @FILE support... @
+checking for strip... strip
+checking for ranlib... ranlib
+checking command to parse /usr/bin/nm -B output from gcc object... ok
+checking for sysroot... no
+checking for a working dd... /usr/bin/dd
+checking how to truncate binary pipes... /usr/bin/dd bs=4096 count=1
+checking for mt... no
+checking if : is a manifest tool... no
+checking for stdio.h... yes
+checking for stdlib.h... yes
+checking for string.h... yes
+checking for inttypes.h... yes
+checking for stdint.h... yes
+checking for strings.h... yes
+checking for sys/stat.h... yes
+checking for sys/types.h... yes
+checking for unistd.h... yes
+checking for dlfcn.h... yes
+checking for objdir... .libs
+checking if gcc supports -fno-rtti -fno-exceptions... no
+checking for gcc option to produce PIC... -fPIC -DPIC
+checking if gcc PIC flag -fPIC -DPIC works... yes
+checking if gcc static flag -static works... yes
+checking if gcc supports -c -o file.o... yes
+checking if gcc supports -c -o file.o... (cached) yes
+checking whether the gcc linker (/usr/bin/ld -m elf_x86_64) supports shared libraries... yes
+checking whether -lc should be explicitly linked in... no
+checking dynamic linker characteristics... GNU/Linux ld.so
+checking how to hardcode library paths into programs... immediate
+checking whether stripping libraries is possible... yes
+checking if libtool supports shared libraries... yes
+checking whether to build shared libraries... yes
+checking whether to build static libraries... yes
+checking whether make supports nested variables... (cached) yes
+checking for pkg-config... /usr/bin/pkg-config
+checking pkg-config is at least version 0.9.0... yes
+checking for ar... /usr/bin/ar
+checking for ranlib... /usr/bin/ranlib
+checking for strip... /usr/bin/strip
+checking dependency style of gcc... gcc3
+checking if gcc supports -std=c89 -pedantic -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef -Wno-unused-function -Wno-long-long -Wno-overlength-strings... yes
+checking if gcc supports -fvisibility=hidden... yes
+checking for valgrind/memcheck.h... yes
+checking for x86_64 assembly availability... yes
+checking for CRYPTO... yes
+checking for main in -lcrypto... yes
+checking for EC functions in libcrypto... yes
+configure: ******
+configure: WARNING: experimental build
+configure: Experimental features do not have stable APIs or properties, and may not be safe for production use.
+configure: Building extrakeys module: yes
+configure: Building schnorrsig module: yes
+configure: ******
+checking that generated files are newer than configure... done
+configure: creating ./config.status
+config.status: creating Makefile
+config.status: creating libsecp256k1.pc
+config.status: creating src/libsecp256k1-config.h
+config.status: src/libsecp256k1-config.h is unchanged
+config.status: executing depfiles commands
+config.status: executing libtool commands
+
+Build Options:
+  with ecmult precomp     = yes
+  with external callbacks = no
+  with benchmarks         = yes
+  with tests              = yes
+  with openssl tests      = yes
+  with coverage           = no
+  module ecdh             = no
+  module recovery         = no
+  module extrakeys        = yes
+  module schnorrsig       = yes
+
+  asm                     = x86_64
+  ecmult window size      = 15
+  ecmult gen prec. bits   = 4
+
+  valgrind                = yes
+  CC                      = gcc
+  CFLAGS                  = -O2 -fvisibility=hidden -std=c89 -pedantic -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef -Wno-unused-function -Wno-long-long -Wno-overlength-strings -W -g
+  CPPFLAGS                =  
+  LDFLAGS                 = 
+
+  CC_FOR_BUILD            = gcc
+  CFLAGS_FOR_BUILD        = -O2 -fvisibility=hidden -std=c89 -pedantic -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef -Wno-unused-function -Wno-long-long -Wno-overlength-strings -W -g
+  CPPFLAGS_FOR_BUILD      =  
+  LDFLAGS_FOR_BUILD       = 

doc/speedup-batch/plot.p

@@ -0,0 +1,33 @@
+set style line 80 lt rgb "#808080"
+set style line 81 lt 0
+set style line 81 lt rgb "#808080"
+set grid back linestyle 81
+set border 3 back linestyle 80
+set xtics nomirror
+set ytics nomirror
+set style line 1 lt rgb "#A00000" lw 2 pt 1
+set style line 2 lt rgb "#00A000" lw 2 pt 6
+set style line 3 lt rgb "#5060D0" lw 2 pt 2
+set style line 4 lt rgb "#F25900" lw 2 pt 9
+set key bottom right
+set autoscale
+unset log
+unset label
+set xtic auto
+set ytic auto
+set title "Batch signature verification in libsecp256k1"
+set xlabel "Number of signatures (logarithmic)"
+set ylabel "Verification time per signature in us"
+set grid
+set logscale x
+set mxtics 10
+
+single_val=system("cat single.dat")
+set xrange [1.1:]
+set xtics add ("2" 2)
+set yrange [0.9:]
+set ytics -1,0.1,3
+set ylabel "Speedup over single verification"
+set term png size 800,600
+set output 'speedup-batch.png'
+plot    "batch.dat" using 1:(single_val/$2) with points title "" ls 1

include/secp256k1_schnorrsig.h

@@ -104,6 +104,27 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify(
     const secp256k1_xonly_pubkey *pubkey
 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4);
 
+/** Verifies a set of Schnorr signatures.
+ *
+ * Returns 1 if all succeeded, 0 otherwise. In particular, returns 1 if n_sigs is 0.
+ *
+ *  Args:    ctx: a secp256k1 context object, initialized for verification.
+ *       scratch: scratch space used for the multiexponentiation
+ *  In:      sig: array of pointers to signatures, or NULL if there are no signatures
+ *         msg32: array of pointers to messages, or NULL if there are no signatures
+ *            pk: array of pointers to x-only public keys, or NULL if there are no signatures
+ *        n_sigs: number of signatures in above arrays. Must be below the
+ *                minimum of 2^31 and SIZE_MAX/2. Must be 0 if above arrays are NULL.
+ */
+SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify_batch(
+    const secp256k1_context* ctx,
+    secp256k1_scratch_space *scratch,
+    const unsigned char *const *sig,
+    const unsigned char *const *msg32,
+    const secp256k1_xonly_pubkey *const *pk,
+    size_t n_sigs
+) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2);
+
 #ifdef __cplusplus
 }
 #endif

src/bench_schnorrsig.c

@@ -15,6 +15,7 @@
 
 typedef struct {
     secp256k1_context *ctx;
+    secp256k1_scratch_space *scratch;
     int n;
 
     const secp256k1_keypair **keypairs;
@@ -47,12 +48,36 @@ void bench_schnorrsig_verify(void* arg, int iters) {
     }
 }
 
+void bench_schnorrsig_verify_n(void* arg, int iters) {
+    bench_schnorrsig_data *data = (bench_schnorrsig_data *)arg;
+    int i, j;
+    const secp256k1_xonly_pubkey **pk = (const secp256k1_xonly_pubkey **)malloc(data->n * sizeof(*pk));
+
+    CHECK(pk != NULL);
+    for (j = 0; j < iters/data->n; j++) {
+        for (i = 0; i < data->n; i++) {
+            secp256k1_xonly_pubkey *pk_nonconst = (secp256k1_xonly_pubkey *)malloc(sizeof(*pk_nonconst));
+            CHECK(secp256k1_xonly_pubkey_parse(data->ctx, pk_nonconst, data->pk[i+j]) == 1);
+            pk[i] = pk_nonconst;
+        }
+        CHECK(secp256k1_schnorrsig_verify_batch(data->ctx, data->scratch, &data->sigs[j], &data->msgs[j], pk, data->n));
+        for (i = 0; i < data->n; i++) {
+            free((void *)pk[i]);
+        }
+    }
+    free(pk);
+}
+
 int main(void) {
     int i;
     bench_schnorrsig_data data;
     int iters = get_iters(10000);
 
     data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY | SECP256K1_CONTEXT_SIGN);
+    /* Scratch space size was selected to allow fitting the maximum number of
+     * points for the default iters value into a single ecmult_multi batch. */
+    /* TODO: this value was updated to support 100 times that */
+    data.scratch = secp256k1_scratch_space_create(data.ctx, 700 * 1024 * 1024);
     data.keypairs = (const secp256k1_keypair **)malloc(iters * sizeof(secp256k1_keypair *));
     data.pk = (const unsigned char **)malloc(iters * sizeof(unsigned char *));
     data.msgs = (const unsigned char **)malloc(iters * sizeof(unsigned char *));
@@ -85,6 +110,16 @@ int main(void) {
 
     run_benchmark("schnorrsig_sign", bench_schnorrsig_sign, NULL, NULL, (void *) &data, 10, iters);
     run_benchmark("schnorrsig_verify", bench_schnorrsig_verify, NULL, NULL, (void *) &data, 10, iters);
+    for (i = 1; i <= iters; i = i*1.2 + 1) {
+        char name[64];
+        int divisible_iters;
+        sprintf(name, "schnorrsig_batch_verify_%d", (int) i);
+
+        data.n = i;
+        divisible_iters = iters - (iters % data.n);
+        run_benchmark(name, bench_schnorrsig_verify_n, NULL, NULL, (void *) &data, 3, divisible_iters);
+        fflush(stdout);
+    }
 
     for (i = 0; i < iters; i++) {
         free((void *)data.keypairs[i]);
@@ -97,6 +132,7 @@ int main(void) {
     free(data.msgs);
     free(data.sigs);
 
+    secp256k1_scratch_space_destroy(data.ctx, data.scratch);
     secp256k1_context_destroy(data.ctx);
     return 0;
 }