GraalVM native-image build support

.circleci/config.yml

@@ -29,7 +29,7 @@ jobs:
             - deps-
       - run:
           name: resolve deps
-          command: clojure -P
+          command: clojure -A:dev:test:build -P
       - save_cache:
           key: deps-{{ checksum "deps.edn" }}
           paths:
@@ -59,19 +59,64 @@ jobs:
           at: /home/circleci
       - run:
           name: clean
-          command: clojure -Sthreads 1 -T:build clean
+          command: clojure -T:build clean
       - run:
           name: compile
-          command: clojure -Sthreads 1 -T:build compile
+          command: clojure -T:build compile
       - run:
           name: jar
-          command: clojure -Sthreads 1 -T:build jar
+          command: clojure -T:build jar
       - persist_to_workspace:
           root: /home/circleci/
           paths:
             - .m2
             - .npm
             - replikativ
+  native-image:
+    machine:
+      image: ubuntu-2004:202010-01
+      resource_class: large
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: install graalvm
+          command: |
+            cd /home/circleci
+            wget https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.2.0/graalvm-ce-java17-linux-amd64-22.2.0.tar.gz
+            tar -xzf graalvm-ce-java17-linux-amd64-22.2.0.tar.gz
+            export PATH=/home/circleci/graalvm-ce-java17-22.2.0/bin:$PATH
+            export JAVA_HOME=/home/circleci/graalvm-ce-java17-22.2.0/
+            gu install native-image
+      - run:
+          name: install clojure
+          command: |
+            cd /home/circleci
+            curl -O https://download.clojure.org/install/linux-install-1.11.1.1165.sh
+            chmod +x linux-install-1.11.1.1165.sh
+            ./linux-install-1.11.1.1165.sh --prefix /home/circleci/clojure
+      - run:
+          name: build native-image
+          command: |
+            cd /home/circleci/replikativ
+            export PATH=/home/circleci/graalvm-ce-java17-22.2.0/bin:/home/circleci/clojure/bin:$PATH
+            export JAVA_HOME=/home/circleci/graalvm-ce-java17-22.2.0/
+            ./bin/build-native-image
+      - persist_to_workspace:
+          root: /home/circleci/
+          paths:
+            - replikativ/dhi
+  native-image-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run native-image test
+          command: |
+            cd /home/circleci/replikativ
+            ./bin/run-native-image-tests
+          no_output_timeout: 5m
   backward-compatibility-test:
     executor: tools/clojurecli
     steps:
@@ -96,7 +141,7 @@ jobs:
           at: /home/circleci
       - run:
           name: deploy
-          command: clojure -Sthreads 1 -T:build deploy
+          command: clojure -T:build deploy
 
 workflows:
   build-test-and-deploy:
@@ -108,6 +153,10 @@ workflows:
           context: dockerhub-deploy
           requires:
             - setup
+      - native-image:
+          context: dockerhub-deploy
+          requires:
+            - setup
       - tools/format:
           context: dockerhub-deploy
           requires:
@@ -116,14 +165,22 @@ workflows:
           context: dockerhub-deploy
           requires:
             - build
+            - native-image
+      - native-image-test:
+          context: dockerhub-deploy
+          requires:
+            - build
+            - native-image
       - backward-compatibility-test:
           context: dockerhub-deploy
           requires:
             - build
+            - native-image
       - tools/integrationtest:
           context: dockerhub-deploy
           requires:
             - build
+            - native-image
       - deploy:
           context:
             - dockerhub-deploy
@@ -136,6 +193,7 @@ workflows:
             - tools/format
             - tools/unittest
             - backward-compatibility-test
+            - native-image-test
             - tools/integrationtest
       - tools/release:
           context:
@@ -145,4 +203,4 @@ workflows:
             branches:
               only: main
           requires:
-            - deploy
+            - deploy

.dir-locals.el

@@ -1,2 +1,2 @@
 ((clojure-mode . ((cider-preferred-build-tool . clojure-cli)
-                  (cider-clojure-cli-aliases . ":dev:test"))))
+                  (cider-clojure-cli-aliases . ":dev:test:libdatahike"))))

.gitignore

@@ -28,4 +28,7 @@ out/
 /.settings
 .classpath
 /.clj-kondo
+/datahike
+datahike.build_artifacts.txt
 tests.user.edn
+/libdatahike/target

bin/build-native-image

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+if ! $(builtin type -P native-image &> /dev/null); then
+    echo "PATH does not contain native-image. Make sure to add your GraalVM to it."
+    exit 1
+else
+    clojure -T:build compile
+    clojure -M:native-cli --no-fallback "-J-Xmx4g"
+fi

bin/run-native-image-tests

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+TMPSTORE=/tmp/dh-test-store
+
+trap "rm -rf $TMPSTORE" EXIT
+
+./dhi delete-database bin/testconfig.edn
+./dhi create-database bin/testconfig.edn
+
+./dhi database-exists bin/testconfig.edn
+
+./dhi benchmark conn:bin/testconfig.edn 0 100000 10000
+./dhi transact conn:bin/testconfig.edn '[[:db/add -1 :name "Judea"]]'
+QUERY_OUT=`./dhi query '[:find (count ?e) . :where [?e :name _]]' db:bin/testconfig.edn`
+
+if [ $QUERY_OUT -eq 100001 ]
+then
+  echo "Test successful."
+else
+  echo "Exception: Query did not return correct value."
+fi
+
+# test history input parsing
+./dhi query '[:find (count ?e) . :where [?e :name _]]' history:bin/testconfig.edn
+./dhi query '[:find (count ?e) . :where [?e :name _]]' since:0:bin/testconfig.edn
+./dhi query '[:find (count ?e) . :where [?e :name _]]' asof:0:bin/testconfig.edn
+
+# other calls
+./dhi pull db:bin/testconfig.edn "[:db/id, :name]" "1"
+./dhi pull-many db:bin/testconfig.edn "[:db/id, :name]" "[1]"
+./dhi entity db:bin/testconfig.edn "1"
+./dhi datoms db:bin/testconfig.edn "{:index :eavt :components [1]}"
+./dhi schema db:bin/testconfig.edn
+./dhi reverse-schema db:bin/testconfig.edn
+./dhi metrics db:bin/testconfig.edn
+
+# test serialization
+./dhi query '[:find ?e . :where [?e :name ?n]]' db:myconfig.edn --format cbor >> /tmp/test
+./dhi query '[:find ?i :in $ ?i . :where [?e :name ?n]]' db:myconfig.edn cbor:/tmp/test # => 1
+
+./dhi delete-database bin/testconfig.edn

bin/testconfig.edn

@@ -0,0 +1,4 @@
+{:store  {:backend :file
+          :path "/tmp/dh-test-store" }
+ :keep-history? true
+ :schema-flexibility :read}

build.clj

@@ -11,6 +11,8 @@
 (def class-dir "target/classes")
 (def basis (b/create-basis {:project "deps.edn"}))
 (def jar-file (format "target/%s-%s.jar" (name lib) version))
+(def native-shared-library-file (format "target/%s-%s-native-shared-library.jar"
+                                        (name lib) version))
 
 (defn clean
   [_]
@@ -21,7 +23,8 @@
   (b/javac {:src-dirs ["java"]
             :class-dir class-dir
             :basis basis
-            :javac-opts ["-source" "8" "-target" "8"]}))
+            :javac-opts ["-source" "8" "-target" "8"]})
+  (spit "resources/DATAHIKE_VERSION" version))
 
 (defn jar
   [_]
@@ -37,6 +40,25 @@
   (b/jar {:class-dir class-dir
           :jar-file jar-file}))
 
+(defn native-shared-library [_]
+  (clean nil)
+  (b/delete {:path "libdatahike/target"})
+  (compile nil)
+  (b/copy-dir {:src-dirs   ["src" "resources" "libdatahike/src"]
+               :target-dir class-dir})
+  (let [basis (b/create-basis {:project "deps.edn"
+                               :aliases [:libdatahike]})]
+    (b/compile-clj {:basis     basis
+                    :src-dirs  ["src"]
+                    :class-dir class-dir})
+    (b/compile-clj {:basis     basis
+                    :src-dirs  ["libdatahike/src"]
+                    :class-dir class-dir})
+    (b/uber {:class-dir class-dir
+             :uber-file native-shared-library-file
+             :basis     basis
+             :main      'datahike.cli})))
+
 (defn deploy
   "Don't forget to set CLOJARS_USERNAME and CLOJARS_PASSWORD env vars."
   [_]

deps.edn

@@ -73,12 +73,24 @@
 
            :ffix {:extra-deps {cljfmt/cljfmt {:mvn/version "0.8.0"}}
                   :main-opts ["-m" "cljfmt.main" "fix"]}
-
            :build {:deps {io.github.clojure/tools.build {:git/tag "v0.8.1" :git/sha "7d40500"}
                           slipset/deps-deploy {:mvn/version "0.2.0"}
                           borkdude/gh-release-artifact {:git/url "https://github.com/borkdude/gh-release-artifact"
                                                         :sha "a83ee8da47d56a80b6380cbb6b4b9274048067bd"}
                           babashka/babashka.curl {:mvn/version "0.1.2"}
                           babashka/fs {:mvn/version "0.1.4"}
-                          cheshire/cheshire {:mvn/version "5.10.2"}}
-                   :ns-default build}}}
+                          cheshire/cheshire {:mvn/version "5.11.0"}}
+                   :ns-default build}
+           :libdatahike {:main-opts ["-e" "(set! *warn-on-reflection* true)"]
+                         :extra-paths ["libdatahike/src"]
+                         :extra-deps {cheshire/cheshire {:mvn/version "5.11.0"}}}
+           :native-cli {:main-opts ["-e" "(set! *warn-on-reflection* true)"
+                                    "-m" "clj.native-image" "datahike.cli"
+                                    "--initialize-at-build-time"
+                                    "-H:Name=dhi"]
+                        :jvm-opts  ["-Dclojure.compiler.direct-linking=true"]
+                        :extra-deps
+                        {cheshire/cheshire {:mvn/version "5.11.0"}
+                         clj.native-image/clj.native-image
+                         {:git/url "https://github.com/taylorwood/clj.native-image.git"
+                          :sha     "7708e7fd4572459c81f6a6b8e44c96f41cdd92d4"}}}}}