Introduce a general memoization system

Introduce a general system for memoizing functions. The core of Dune
is refactored on top of this system. The new system will allow to
share not only the result of external commands between builds but also
the result of internal computations. The system automatically keeps
tracks of the effects performed by computations in order to detect
when it is safe to reuse previous results.

The intents of this change is to make incremental builds faster,
especially in polling mode. The new system is more powerful as well
and will allow us to use the result of commands for computing rules.

This patch also re-implement cycle detection as we realized that the
previous algorithm was broken and the problem showed up more often in
the new system. The new algorithm was taken from the following paper:

  Michael A. Bender, Jeremy T. Fineman, Seth Gilbert, and Robert
  E. Tarjan. 2015. A New Approach to Incremental Cycle Detection and
  Related Problems. ACM Trans. Algorithms 12, 2, Article 14 (December
  2015), 22 pages. DOI: https://doi.org/10.1145/2756553 *)

Signed-off-by: Rudi Horn <dyn-git@rudi-horn.de>
Signed-off-by: Jeremie Dimino <jeremie@dimino.org>

CHANGES.md

@@ -13,6 +13,13 @@ unreleased
 - Fix preprocessing for libraries with `(include_subdirs ..)` (#1624, fix #1626,
   @nojb, @rgrinberg)
 
+- Reimplement the core of Dune using a new generic memoization system
+  (#1489, @rudihorn, @diml)
+
+- Replace the broken cycle detection algorithm by a state of the art
+  one from [this paper](https://doi.org/10.1145/2756553) (#1489,
+  @rudihorn)
+
 1.6.2 (05/12/2018)
 ------------------
 

bin/main.ml

@@ -264,13 +264,14 @@ let external_lib_deps =
     in
     Common.set_common common ~targets:[];
     let log = Log.create common in
-    let setup =
+    let setup, lib_deps =
       Scheduler.go ~log ~common
-        (Main.setup ~log common ~external_lib_deps_mode:true)
+        (Main.setup ~log common ~external_lib_deps_mode:true >>= fun setup ->
+         let targets = Target.resolve_targets_exn ~log common setup targets in
+         let request = Target.request setup targets in
+         Build_system.all_lib_deps setup.build_system ~request >>| fun deps ->
+         (setup, deps))
     in
-    let targets = Target.resolve_targets_exn ~log common setup targets in
-    let request = Target.request setup targets in
-    let lib_deps = Build_system.all_lib_deps setup.build_system ~request in
     let failure =
       String.Map.foldi lib_deps ~init:false
         ~f:(fun context_name lib_deps_by_dir acc ->

bootstrap.ml

@@ -34,6 +34,8 @@ let dirs =
   ; "src/stdune"             , Some "Stdune"
   ; "src/fiber"              , Some "Fiber"
   ; "src/xdg"                , Some "Xdg"
+  ; "src/dag"                , Some "Dag"
+  ; "src/memo"               , Some "Memo"
   ; "src/ocaml-config"       , Some "Ocaml_config"
   ; "vendor/boot"            , None
   ; "src/dune_lang"              , Some "Dune_lang"

example/sample-projects/hello_world/run.t

@@ -2,15 +2,15 @@
       ocamldep bin/.main.eobjs/main.ml.d
       ocamldep lib/.hello_world.objs/hello_world.ml.d
         ocamlc lib/.hello_world.objs/hello_world.{cmi,cmo,cmt}
+        ocamlc lib/hello_world.cma
+      ocamldep test/.test.eobjs/test.ml.d
       ocamlopt lib/.hello_world.objs/hello_world.{cmx,o}
       ocamlopt lib/hello_world.{a,cmxa}
-      ocamlopt lib/hello_world.cmxs
-      ocamldep test/.test.eobjs/test.ml.d
         ocamlc test/.test.eobjs/test.{cmi,cmo,cmt}
       ocamlopt test/.test.eobjs/test.{cmx,o}
       ocamlopt test/test.exe
           test test/test.output
-        ocamlc lib/hello_world.cma
+      ocamlopt lib/hello_world.cmxs
         ocamlc bin/.main.eobjs/main.{cmi,cmo,cmt}
       ocamlopt bin/.main.eobjs/main.{cmx,o}
       ocamlopt bin/main.exe

example/sample-projects/with-configure-step/run.t

@@ -3,7 +3,7 @@
       ocamldep src/.plop.eobjs/config.ml.d
       ocamldep src/.plop.eobjs/plop.ml.d
         ocamlc src/.plop.eobjs/config.{cmi,cmo,cmt}
-      ocamlopt src/.plop.eobjs/config.{cmx,o}
         ocamlc src/.plop.eobjs/plop.{cmi,cmo,cmt}
       ocamlopt src/.plop.eobjs/plop.{cmx,o}
+      ocamlopt src/.plop.eobjs/config.{cmx,o}
       ocamlopt src/plop.exe