Add module dlopen()/LoadLibrary() support

Trivial test included.

TODO:

 - write more docs
 - write more tests
 - write some interesting sample modules (e.g., access to the system's
   RNG, simple file I/O, system(), and popen().

Makefile.am

@@ -59,7 +59,7 @@ libjq_la_LIBADD += -lshlwapi
 libjq_la_LDFLAGS += -no-undefined
 endif
 
-include_HEADERS = src/jv.h src/jq.h
+include_HEADERS = src/jv.h src/jq.h src/jq_plugin.h
 
 if ENABLE_UBSAN
 AM_CFLAGS += -fsanitize=undefined
@@ -178,6 +178,7 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/modules/lib/jq/e/e.jq tests/modules/lib/jq/f.jq           \
         tests/modules/shadow1.jq tests/modules/shadow2.jq               \
         tests/modules/syntaxerror/syntaxerror.jq                        \
+        tests/modules/somod/somod.jq tests/modules/somod/somod.c        \
         tests/modules/test_bind_order.jq                                \
         tests/modules/test_bind_order0.jq                               \
         tests/modules/test_bind_order1.jq                               \

README.plugins

@@ -0,0 +1,75 @@
+jq FFI
+======
+
+jq now has an FFI.  You can create a module with C-coded builtins.
+
+A jq module's jq source can say:
+
+    module {"cfunctions":true};
+
+to cause a shared object or DLL of the same name to be loaded.
+
+An alternative shared object or DLL name can be specified instead:
+
+    module {"cfunctions":"my_shared_object"};
+
+The shared object or DLL must export a function named `jq_plugin_init`,
+or else it must list an alternative name for that function it its module
+meta:
+
+    module {"cfunctions":true,"plugin_init_function":"my_plugin_init"};
+
+The module's C sources must `#define JQ_PLUGIN` and must `#include`
+`<jv.h>`, `<jq.h>`, and `<jq_plugin.h>`.
+
+The init function has the following prototype:
+
+    typedef int (*jq_plugin_init_f)
+        (int,                       /* jq plugin min supported ABI version */
+         int,                       /* jq plugin current       ABI version */
+         struct jq_state *,
+         const char **,             /* error string */
+         const char **,             /* jq-coded module contents */
+         size_t *,                  /* jq-coded module contents size */
+         struct cfunction **,       /* array of C-coded function descriptors */
+         size_t *);
+
+The plugin must check that two jq ABI integers include the
+`JQ_CURRENT_ABI` that the plugin was compiled against.
+
+The plugin can output a replacement for its jq-coded part via the fifth
+and sixth arguments.  If the seventh argument is `0`, then the fifth
+output, if non-`NULL`, must be a C string (i.e., `NUL`-terminated).
+
+The plugin can output C-coded functions via the eigth and ninth
+arguments.
+
+Note that all `jv` and `jq` functions in a plugin are replaced with
+macros of this form:
+
+    #define fname ((*(struct jq_plugin_vtable **)jq)->fname)
+
+Here `jq` should be the `struct jq_state *` argument to the C-coded
+builtin functions and the module's init function.
+
+C-coded functions must have the following form:
+
+    jv my_func(struct jq_state *jq, jv input);
+    jv my_func(struct jq_state *jq, jv input, jv arg);
+    jv my_func(struct jq_state *jq, jv input, jv a0, jv a1);
+    ..
+    jv my_func(struct jq_state *jq, jv input, jv a0, jv a1, jv a2, jv a3, jv a4, jv a5);
+
+and must have an entry in the `struct cfunction` array output by the
+module's init function.
+
+`struct cfunction` is:
+
+    typedef void (*cfunction_ptr)();
+    struct cfunction {
+      cfunction_ptr fptr; /* function pointer */
+      const char* name;   /* jq function name */
+      int nargs;          /* number of arguments; must be at least 1 */
+      int pure;           /* 1 if pure, 0 if impure */
+    };
+

configure.ac

@@ -86,6 +86,10 @@ dnl Enable building all static
 AC_ARG_ENABLE([all-static],
    AC_HELP_STRING([--enable-all-static], [link jq with static libraries only]))
 
+if test "x$enable_all_static" != xyes; then
+  AC_FIND_FUNC([dlopen], [dl], [#include <dlfcn.h>], [0, 0])
+fi
+
 AS_IF([test "x$enable_docs" != "xno"],[
    AC_CHECK_PROGS(pipenv_cmd, pipenv)
 

scripts/dwarf2h

@@ -0,0 +1,233 @@
+#!/bin/bash
+
+PROG=${0##*/}
+
+function usage {
+  ((${1:-1})) && exec 1>&2
+  cat <<EOF
+Usage: $PROG [options] OBJECT...
+       $PROG [options] -i DWARF_DUMP_FILE
+  Prints C prototypes and other information about functions exported
+  from OBJECT.  The OBJECT files must be compiled to contain DWARF
+  sections -- i.e., use -ggdb3.
+
+  (The second usage is intended for debugging.)
+
+  This is intended to be used for generating v-tables for plugins.
+
+  Output options:
+  
+    -B      Output function stubs for exported functions
+    -I      Generate code to initialize a v-table
+    -P      Output prototypes for exported functions
+    -n      Include argument names in prototypes
+    -T      Output function pointer typedefs for exported functions
+
+    -s NAME Name of v-table struct
+    -S      Output struct definition (requires -s NAME)
+    -M      Output macros that indirect through jq_state v-table
+            (requires -s NAME)
+
+  Function symbol filter options:
+
+    -p GLOB Only include functions whose names match GLOB
+            (may be given multiple times)
+    -x GLOB Exclude functions matching GLOB
+            (may be given multiple times)
+    -f FILE Exclude functions that appear to be in a v-table in FILE
+            (useful when updating a v-table so as to generate additions
+             to the v-table that go at the end)
+            (may be given multiple times)
+
+  Debug options:
+
+    -i FILE Use the dump of DWARF data in FILE, made with llvm-dwarfdump
+    -v      Trace this script
+
+  At least one of -P, -T, -S, or -s must be given.
+EOF
+  exit ${1:-1}
+}
+
+(($#==0)) && usage
+declare -A exclude
+pats=()
+xpats=()
+dwarf_in=
+struct_name=
+stubs=false
+struct=false
+macros=false
+typedefs=false
+prototypes=false
+initializers=false
+include_arg_names=false
+while getopts +:IMPSTf:bhi:np:s:vx: opt; do
+  case "$opt" in
+  I) initializers=true;;
+  M) macros=true;;
+  P) prototypes=true;;
+  S) struct=true;;
+  T) typedefs=true;;
+  f) for fname in $(
+            grep '^  \([^ ]*\)_f \1;' "$OPTARG" | cut -d' ' -f4 | cut -d';' -f1;
+            grep ')(' "$OPTARG" | cut -d'(' -f2 | cut -d'*' -f2 | cut -d')' -f1
+           ); do
+       exclude[$fname]=true
+     done ;;
+  b) stubs=true;;
+  h) usage 0;;
+  i) dwarf_in=$OPTARG;;
+  n) include_arg_names=true;;
+  p) pats+=("$OPTARG");;
+  s) struct_name=$OPTARG;;
+  v) set -vx;;
+  x) xpats+=("$OPTARG");;
+  *) usage;;
+  esac
+done
+shift $((OPTIND - 1))
+(($#==0)) && [[ -z $dwarf_in ]] && usage
+(($#)) && [[ -n $dwarf_in ]] && usage
+object=$1
+shift
+
+! $stubs && ! $typedefs && ! $prototypes && [[ -z $struct_name ]] && usage
+
+$struct && [[ -z $struct_name ]] && usage
+$macros && [[ -z $struct_name ]] && usage
+
+declare -A fnames
+fields=()
+OIFS=$IFS
+while read a b; do
+  [[ $a = 0x* && $b = DW_TAG_subprogram ]] || continue
+  args=()
+  argnames=()
+  argtypes=()
+  fname=
+  rettype=void
+  exported=false
+  # Extract function name and type
+  while read a b; do
+    [[ $a = 0x* ]] && break
+    case "$a" in
+    DW_AT_external) [[ $b = *true* ]] && exported=true;;
+    DW_AT_name)
+      b=${b%\"*}
+      fname=${b#*\"};;
+    DW_AT_type)
+      b=${b%\"*}
+      rettype=${b#*\"}
+      [[ $rettype = [*]* ]] && rettype="void $rettype";;
+    *) true;;
+    esac
+  done
+  skip=false
+  [[ $rettype = *subroutine* ]] && skip=true
+  while [[ $b = DW_TAG_formal_parameter || $b = DW_TAG_unspecified_parameters ]]; do
+    if [[ $b = DW_TAG_unspecified_parameters ]]; then
+      argtypes+=(...)
+      read a b
+      continue
+    fi
+    argname=
+    argtype=void
+    while read a b; do
+      [[ $a = 0x* ]] && break
+      case "$a" in
+      DW_AT_name)
+        b=${b%\"*}
+        argname=${b#*\"}
+        argnames+=("$argname");;
+      DW_AT_type)
+        b=${b%\"*}
+        argtype=${b#*\"}
+        [[ $argtype = [*]* ]] && argtype="void $argtype"
+        [[ $argtype = va_list ]] && argtype=...
+        [[ $argtype = __va_list_tag\* ]] && argtype=va_list
+        [[ $argtype = *subroutine* ]] && skip=true
+        argtypes+=("$argtype");;
+      *) true;;
+      esac
+    done
+    args+=("$argtype $argname")
+  done
+  while [[ $a != 0x* || $b != NULL ]]; do
+    read a b
+  done
+  if $skip; then
+    printf 'WARNING: Excluding function with function-valued return type or argument, %s()\n' "$fname" 1>&2
+    continue
+  fi
+  [[ -n ${fnames[$fname]} ]] && continue # probably static inline
+  $exported || continue # static
+  if ((${#pats[@]})); then
+    match=false
+    for pat in "${pats[@]}"; do
+      if [[ $fname = $pat ]]; then
+        match=true
+        break
+      fi
+    done
+    $match || continue
+  fi
+  if ((${#xpats[@]})); then
+    match=false
+    for pat in "${xpats[@]}"; do
+      if [[ $fname = $pat ]]; then
+        match=true
+        break
+      fi
+    done
+    $match && continue
+  fi
+  [[ -z $fname ]] && continue
+  [[ -n ${exclude[$fname]} ]] && continue
+  fnames[$fname]=$fname
+  IFS=,
+  $typedefs && printf 'typedef %s (*%s_f)(%s);\n' "$rettype" "$fname" "${argtypes[*]}"
+  $prototypes && $include_arg_names &&
+    printf '%s %s(%s);\n' "$rettype" "$fname" "${args[*]}"
+  $prototypes && ! $include_arg_names &&
+    printf '%s %s(%s);\n' "$rettype" "$fname" "${argtypes[*]}"
+  $stubs && printf '%s\n%s(%s)\n{\n}\n' "$rettype" "$fname" "${args[*]}"
+  if $typedefs; then
+    fields+=("${fname}_f ${fname}")
+  else
+    fields+=("$rettype (*${fname})(${argtypes[*]})")
+  fi
+  IFS=$OIFS
+done < <(if [[ -n $dwarf_in ]]; then cat "$dwarf_in"; else llvm-dwarfdump-6.0 "$@"; fi)
+
+if $struct; then
+  printf 'struct %s {\n' "$struct_name"
+  if ((${#exclude[@]} == 0)); then
+    printf '  %s;\n' "${fields[@]}"
+  else
+    # Print only fields for functions not listed in the FILE given via -f FILE.
+    #
+    # The developer will have to edit that file and add the new fields (and
+    # typedefs) themselves.
+    for field in "${fields[@]}"; do
+      fname=$(echo "$field" | grep '^\([^ ]*\)_f \1' | cut -d' ' -f2 | cut -d';' -f1;
+              echo "$field" | grep ')(' | cut -d'(' -f2 | cut -d'*' -f2 | cut -d')' -f1)
+      [[ -n ${exclude[$fname]} ]] && continue
+      printf '  %s;\n' "$field"
+    done
+  fi
+  printf '};\n'
+fi
+
+if $macros; then
+  for fname in "${!fnames[@]}"; do
+    [[ -n ${exclude[$fname]} ]] && continue
+    printf '#define %s ((*(struct %s **)jq)->%s)\n' "$fname" "$struct_name" "$fname"
+  done
+fi
+
+if $initializers; then
+  for fname in "${!fnames[@]}"; do
+    printf 'vtable->%s = %s;\n' "$fname" "$fname"
+  done
+fi

src/bytecode.h

@@ -3,6 +3,8 @@
 #include <stdint.h>
 
 #include "jv.h"
+#include "jq.h"
+#include "jq_plugin.h"
 
 typedef enum {
 #define OP(name, imm, in, out) name,
@@ -43,15 +45,6 @@ struct opcode_description {
 
 const struct opcode_description* opcode_describe(opcode op);
 
-
-#define MAX_CFUNCTION_ARGS 10
-typedef void (*cfunction_ptr)();
-struct cfunction {
-  cfunction_ptr fptr;
-  const char* name;
-  int nargs;
-};
-
 struct symbol_table {
   struct cfunction* cfunctions;
   int ncfunctions;