Introduce a general memoization system

Introduce a general system for memoizing functions. The core of Dune
is refactored on top of this system. The new system will allow to
share not only the result of external commands between builds but also
the result of internal computations. The system automatically keeps
tracks of the effects performed by computations in order to detect
when it is safe to reuse previous results.

The intents of this change is to make incremental builds faster,
especially in polling mode. The new system is more powerful as well
and will allow us to use the result of commands for computing rules.

This patch also re-implement cycle detection as we realized that the
previous algorithm was broken and the problem showed up more often in
the new system. The new algorithm was taken from the following paper:

  Michael A. Bender, Jeremy T. Fineman, Seth Gilbert, and Robert
  E. Tarjan. 2015. A New Approach to Incremental Cycle Detection and
  Related Problems. ACM Trans. Algorithms 12, 2, Article 14 (December
  2015), 22 pages. DOI: https://doi.org/10.1145/2756553 *)

Signed-off-by: Rudi Horn <dyn-git@rudi-horn.de>
Signed-off-by: Jeremie Dimino <jeremie@dimino.org>

CHANGES.md

@@ -35,6 +35,13 @@ next
 
 - Document virtual libraries (experimental) (#1580, @rgrinberg)
 
+- Reimplement the core of Dune using a new generic memoization system
+  (#1489, @rudihorn, @diml)
+
+- Replace the broken cycle detection algorithm by a state of the art
+  one from [this paper](https://doi.org/10.1145/2756553) (#1489,
+  @rudihorn)
+
 1.5.1 (7/11/2018)
 -----------------
 

bin/main.ml

@@ -258,9 +258,9 @@ let external_lib_deps =
        >>= fun setup ->
        let targets = Target.resolve_targets_exn ~log common setup targets in
        let request = Target.request setup targets in
-       let failure =
+       Build_system.all_lib_deps_by_context setup.build_system ~request
+       >>|
          String.Map.foldi ~init:false
-           (Build_system.all_lib_deps_by_context setup.build_system ~request)
            ~f:(fun context_name lib_deps acc ->
              let internals =
                Super_context.internal_lib_names
@@ -319,7 +319,7 @@ let external_lib_deps =
                  (format_external_libs externals);
                acc
              end)
-       in
+       >>= fun failure ->
        if failure then raise Already_reported;
        Fiber.return ())
   in

bootstrap.ml

@@ -34,6 +34,8 @@ let dirs =
   ; "src/stdune"             , Some "Stdune"
   ; "src/fiber"              , Some "Fiber"
   ; "src/xdg"                , Some "Xdg"
+  ; "src/dag"                , Some "Dag"
+  ; "src/memo"               , Some "Memo"
   ; "src/ocaml-config"       , Some "Ocaml_config"
   ; "vendor/boot"            , None
   ; "src/dune_lang"              , Some "Dune_lang"

example/sample-projects/hello_world/run.t

@@ -2,15 +2,15 @@
       ocamldep bin/.main.eobjs/main.ml.d
       ocamldep lib/.hello_world.objs/hello_world.ml.d
         ocamlc lib/.hello_world.objs/hello_world.{cmi,cmo,cmt}
+        ocamlc lib/hello_world.cma
+      ocamldep test/.test.eobjs/test.ml.d
       ocamlopt lib/.hello_world.objs/hello_world.{cmx,o}
       ocamlopt lib/hello_world.{a,cmxa}
-      ocamlopt lib/hello_world.cmxs
-      ocamldep test/.test.eobjs/test.ml.d
         ocamlc test/.test.eobjs/test.{cmi,cmo,cmt}
       ocamlopt test/.test.eobjs/test.{cmx,o}
       ocamlopt test/test.exe
           test test/test.output
-        ocamlc lib/hello_world.cma
+      ocamlopt lib/hello_world.cmxs
         ocamlc bin/.main.eobjs/main.{cmi,cmo,cmt}
       ocamlopt bin/.main.eobjs/main.{cmx,o}
       ocamlopt bin/main.exe

example/sample-projects/with-configure-step/run.t

@@ -3,7 +3,7 @@
       ocamldep src/.plop.eobjs/config.ml.d
       ocamldep src/.plop.eobjs/plop.ml.d
         ocamlc src/.plop.eobjs/config.{cmi,cmo,cmt}
-      ocamlopt src/.plop.eobjs/config.{cmx,o}
         ocamlc src/.plop.eobjs/plop.{cmi,cmo,cmt}
       ocamlopt src/.plop.eobjs/plop.{cmx,o}
+      ocamlopt src/.plop.eobjs/config.{cmx,o}
       ocamlopt src/plop.exe