Add deinitializeDMD function to deinitialize the front end

This is a start of being able to restore the global state initialized
by `initDMD` to its original state. It only deinitializes state
initialized by `initDMD`, state store directly inside functions are
not restored.

This is useful to invoke the compiler multiple times within the same
application. This allows to write more fine grained tests for the
compiler, more of a unit test style compared to the current end to end
tests.

dub.sdl

@@ -33,19 +33,14 @@ subPackage {
   preGenerateCommands `
     "$${DUB_EXE}" \
     --arch=$${DUB_ARCH} \
+    --compiler=$${DC} \
     --single "$${DUB_PACKAGE_DIR}config.d" \
     -- "$${DUB_PACKAGE_DIR}generated/dub" \
     "$${DUB_PACKAGE_DIR}VERSION" \
     /etc
    ` platform="posix"
 
-  preGenerateCommands `
-    "%DUB_EXE%" ^
-    --arch=%DUB_ARCH% ^
-    --single "%DUB_PACKAGE_DIR%config.d" ^
-    -- "%DUB_PACKAGE_DIR%generated/dub" ^
-    "%DUB_PACKAGE_DIR%VERSION"
-  ` platform="windows"
+  preGenerateCommands `"%DUB_EXE%" --arch=%DUB_ARCH% --compiler="%DC%" --single "%DUB_PACKAGE_DIR%config.d" -- "%DUB_PACKAGE_DIR%generated/dub" "%DUB_PACKAGE_DIR%VERSION"` platform="windows"
 
   stringImportPaths "generated/dub"
 

src/dmd/builtin.d

@@ -425,6 +425,17 @@ public extern (C++) void builtin_init()
         add_builtin("_D4core5bitop7_popcntFNaNbNiNfmZi", &eval_popcnt);
 }
 
+/**
+ * Deinitializes the global state of the compiler.
+ *
+ * This can be used to restore the state set by `builtin_init` to its original
+ * state.
+ */
+public void builtinDeinitialize()
+{
+    builtins = builtins.init;
+}
+
 /**********************************
  * Determine if function is a builtin one that we can
  * evaluate at compile time.

src/dmd/dmodule.d

@@ -299,6 +299,17 @@ extern (C++) final class Module : Package
         modules = new DsymbolTable();
     }
 
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `_init` to its original
+     * state.
+     */
+    static void deinitialize()
+    {
+        modules = modules.init;
+    }
+
     extern (C++) __gshared AggregateDeclaration moduleinfo;
 
     const(char)* arg;           // original argument name

src/dmd/expression.d

@@ -640,6 +640,22 @@ extern (C++) abstract class Expression : RootObject
         CTFEExp.showcontext = new CTFEExp(TOK.showCtfeContext);
     }
 
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `_init` to its original
+     * state.
+     */
+    static void deinitialize()
+    {
+        CTFEExp.cantexp = CTFEExp.cantexp.init;
+        CTFEExp.voidexp = CTFEExp.voidexp.init;
+        CTFEExp.breakexp = CTFEExp.breakexp.init;
+        CTFEExp.continueexp = CTFEExp.continueexp.init;
+        CTFEExp.gotoexp = CTFEExp.gotoexp.init;
+        CTFEExp.showcontext = CTFEExp.showcontext.init;
+    }
+
     /*********************************
      * Does *not* do a deep copy.
      */

src/dmd/frontend.d

@@ -76,6 +76,35 @@ void initDMD()
     FileCache._init();
 }
 
+/**
+Deinitializes the global variables of the DMD compiler.
+
+This can be used to restore the state set by `initDMD` to its original state.
+Useful if there's a need for multiple sessions of the DMD compiler in the same
+application.
+*/
+void deinitializeDMD()
+{
+    import dmd.builtin : builtinDeinitialize;
+    import dmd.dmodule : Module;
+    import dmd.expression : Expression;
+    import dmd.globals : global;
+    import dmd.id : Id;
+    import dmd.mtype : Type;
+    import dmd.objc : Objc;
+    import dmd.target : target;
+
+    global.deinitialize();
+
+    Type.deinitialize();
+    Id.deinitialize();
+    Module.deinitialize();
+    target.deinitialize();
+    Expression.deinitialize();
+    Objc.deinitialize();
+    builtinDeinitialize();
+}
+
 /**
 Add import path to the `global.path`.
 Params:

src/dmd/globals.d

@@ -401,6 +401,17 @@ struct Global
         params.color = Console.detectTerminal();
     }
 
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `_init` to its original
+     * state.
+     */
+    void deinitialize()
+    {
+        this = this.init;
+    }
+
     /**
     Returns: the version as the number that would be returned for __VERSION__
     */

src/dmd/id.d

@@ -39,6 +39,17 @@ struct Id
     {
         mixin(msgtable.generate(&initializer));
     }
+
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `initialize` to its original
+     * state.
+     */
+    void deinitialize()
+    {
+        mixin(msgtable.generate(&deinitializer));
+    }
 }
 
 private:
@@ -487,3 +498,9 @@ string initializer(Msgtable m)
 {
     return m.ident ~ ` = Identifier.idPool("` ~ m.name ~ `");`;
 }
+
+// Used to generate the code for each deinitializer.
+string deinitializer(Msgtable m)
+{
+    return m.ident ~ " = Identifier.init;";
+}

src/dmd/mtype.d

@@ -905,6 +905,17 @@ extern (C++) abstract class Type : RootObject
         thash_t = tsize_t;
     }
 
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `_init` to its original
+     * state.
+     */
+    static void deinitialize()
+    {
+        stringtable = stringtable.init;
+    }
+
     final d_uns64 size()
     {
         return size(Loc.initial);

src/dmd/objc.d

@@ -188,6 +188,17 @@ extern(C++) abstract class Objc
             _objc = new Unsupported;
     }
 
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `_init` to its original
+     * state.
+     */
+    static void deinitialize()
+    {
+        _objc = _objc.init;
+    }
+
     abstract void setObjc(ClassDeclaration cd);
     abstract void setObjc(InterfaceDeclaration);
 

src/dmd/target.d

@@ -180,6 +180,17 @@ struct Target
             params.isDragonFlyBSD || params.isOSX;
     }
 
+    /**
+     * Deinitializes the global state of the compiler.
+     *
+     * This can be used to restore the state set by `_init` to its original
+     * state.
+     */
+    void deinitialize()
+    {
+        this = this.init;
+    }
+
     /**
      * Requested target memory alignment size of the given type.
      * Params:

test/Makefile

@@ -143,6 +143,9 @@ test_tools=$(RESULTS_DIR)/d_do_test$(EXE) $(RESULTS_DIR)/sanitize_json$(EXE)
 $(RESULTS_DIR)/%.out: % $(RESULTS_DIR)/.created $(test_tools) $(DMD)
 	$(QUIET) $(RESULTS_DIR)/d_do_test $<
 
+$(RESULTS_DIR)/%.out: % $(RESULTS_DIR)/.created $(RESULTS_DIR)/unit_test_runner$(EXE) $(DMD)
+	$(QUIET) $(RESULTS_DIR)/d_do_test $<
+
 quick:
 	$(MAKE) ARGS="" run_tests
 
@@ -158,7 +161,11 @@ $(RESULTS_DIR)/.created:
 	$(QUIET)if [ ! -d $(RESULTS_DIR)/fail_compilation ]; then mkdir $(RESULTS_DIR)/fail_compilation; fi
 	$(QUIET)touch $(RESULTS_DIR)/.created
 
-run_tests: start_runnable_tests start_compilable_tests start_fail_compilation_tests
+run_tests: unit_tests start_runnable_tests start_compilable_tests start_fail_compilation_tests
+
+unit_tests: $(RESULTS_DIR)/unit_test_runner$(EXE)
+	@echo "Running unit tests"
+	$<
 
 run_runnable_tests: $(runnable_test_results)
 
@@ -193,3 +200,9 @@ $(RESULTS_DIR)/sanitize_json$(EXE): tools/sanitize_json.d $(RESULTS_DIR)/.create
 	@echo "PIC: '$(PIC_FLAG)'"
 	$(DMD) -conf= $(MODEL_FLAG) $(DEBUG_FLAGS) -od$(RESULTS_DIR) -of$(RESULTS_DIR)$(DSEP)sanitize_json$(EXE) -i $<
 
+$(RESULTS_DIR)/unit_test_runner$(EXE): tools/unit_test_runner.d $(RESULTS_DIR)/.created
+	@echo "Building unit_test_runner tool"
+	@echo "OS: '$(OS)'"
+	@echo "MODEL: '$(MODEL)'"
+	@echo "PIC: '$(PIC_FLAG)'"
+	$(DMD) -conf= $(MODEL_FLAG) $(DEBUG_FLAGS) -od$(RESULTS_DIR) -of$(RESULTS_DIR)$(DSEP)unit_test_runner$(EXE) -i $<

test/README.md

@@ -65,6 +65,65 @@ Note:
 - `AUTO_UPDATE` doesn't work with tests that have multiple `TEST_OUTPUT` segments
 - `AUTO_UPDATE` can be set as an environment variable or as Makefile-like argument assignment
 
+### Running the Unit Tests
+
+The unit tests will automatically run when all tests are run using `./run.d` or
+`make`. To only run the unit tests the `./run.d unit_tests` command can be used.
+
+For more finer grain control over the unit tests the `./unit_test_runnner.d`
+command can be used:
+
+To run all unit tests:
+
+```sh
+./unit_test_runnner.d
+```
+
+To only run the unit tests in one or more specific files:
+
+```sh
+./unit_test_runnner.d unit/deinitialization.d
+```
+
+To only run a subset of the unit tests in a single file:
+
+```sh
+./unit_test_runnner.d unit/deinitialization.d --filter Expression
+```
+
+In the above example, the `--filter` (or `-f`) flag will filter to only run the
+tests with a UDA matching the given value, in this case `Expression`.
+
+```d
+@("Target.deinitialize")
+unittest {}
+
+@("Expression.deinitialize")
+unittest {}
+```
+
+Of the above unit tests, only the second one will be run, since
+`-filter Expression` was specified.
+
+The `--filter` flag works when no files are specified as well.
+
+## Types of Tests
+
+There are two types of tests in the DMD test suite:
+
+* **End-to-end test**. These are tests that invokes the compiler as an external
+process in some kind of way. Then it asserts either the exit code or the output
+of the compiler. These tests are located in `compilable`, `fail_compilation` and
+`runnable`.
+
+* **Unit tests**. These tests are more of a unit test, integration or
+functional style tests. These tests are using the compiler as a library. They
+are more flexible because they can assert state internal to the compiler which
+the end-to-end tests would never have access to. The unit test runner will
+compile all files in the `unit` directory into a single executable and run the
+tests. This should make it quick to run the tests since only a single process
+need to be started.
+
 Makefile targets
 ----------------
 
@@ -74,6 +133,7 @@ Makefile targets
     run_runnable_tests:         run just the runnable tests
     run_compilable_tests:       run just the compilable tests
     run_fail_compilation_tests: run just the fail compilation tests
+    unit_test:                  run all unit tests (those in the "unit" directory)
 
     quick:              run all tests with no default permuted args
                         (individual test specified options still honored)