Use cache_dir for coconut-run and auto comp

Resolves   #768.

.gitignore

@@ -131,12 +131,13 @@ __pypackages__/
 .vscode
 
 # Coconut
-coconut/tests/dest/
-docs/
+/coconut/tests/dest/
+/docs/
 pyston/
 pyprover/
 bbopt/
 coconut-prelude/
 index.rst
 vprof.json
-coconut/icoconut/coconut/
+/coconut/icoconut/coconut/
+__coconut_cache__/

DOCS.md

@@ -235,7 +235,9 @@ which will quietly compile and run `<source>`, passing any additional arguments
 
 To pass additional compilation arguments to `coconut-run` (e.g. `--no-tco`), put them before the `<source>` file.
 
-`coconut-run` will always use [automatic compilation](#automatic-compilation), such that Coconut source files can be directly imported from any Coconut files run via `coconut-run`. Additionally, compilation parameters (e.g. `--no-tco`) used in `coconut-run` will be passed along and used for any auto compilation.
+`coconut-run` will always enable [automatic compilation](#automatic-compilation), such that Coconut source files can be directly imported from any Coconut files run via `coconut-run`. Additionally, compilation parameters (e.g. `--no-tco`) used in `coconut-run` will be passed along and used for any auto compilation.
+
+On modern Python versions, `coconut-run` will use a `__coconut_cache__` directory to cache the compiled Python. Note that `__coconut_cache__` will always be removed from `__file__`.
 
 #### Naming Source Files
 
@@ -4394,7 +4396,7 @@ Automatic compilation lets you simply import Coconut files directly without havi
 
 Once automatic compilation is enabled, Coconut will check each of your imports to see if you are attempting to import a `.coco` file and, if so, automatically compile it for you. Note that, for Coconut to know what file you are trying to import, it will need to be accessible via `sys.path`, just like a normal import.
 
-Automatic compilation always compiles modules and packages in-place, and compiles with `--target sys --line-numbers --keep-lines` by default.
+Automatic compilation always compiles with `--target sys --line-numbers --keep-lines` by default. On modern Python versions, automatic compilation will use a `__coconut_cache__` directory to cache the compiled Python. Note that `__coconut_cache__` will always be removed from `__file__`.
 
 Automatic compilation is always available in the Coconut interpreter or when using [`coconut-run`](#coconut-scripts). When using auto compilation through the Coconut interpreter, any compilation options passed in will also be used for auto compilation. Additionally, the interpreter always allows importing from the current working directory, letting you easily compile and play around with a `.coco` file simply by running the Coconut interpreter and importing it.
 
@@ -4529,12 +4531,14 @@ Retrieves a string containing information about the Coconut version. The optiona
 
 #### `auto_compilation`
 
-**coconut.api.auto_compilation**(_on_=`True`, _args_=`None`)
+**coconut.api.auto_compilation**(_on_=`True`, _args_=`None`, _use\_cache\_dir_=`None`)
 
 Turns [automatic compilation](#automatic-compilation) on or off. This function is called automatically when `coconut.api` is imported.
 
 If _args_ is passed, it will set the Coconut command-line arguments to use for automatic compilation. Arguments will be processed the same way as with [`coconut-run`](#coconut-scripts) such that `--quiet --target sys --keep-lines` will all be set by default.
 
+If _use\_cache\_dir_ is passed, it will turn on or off the usage of a `__coconut_cache__` directory to put compile files in rather than compiling them in-place. Note that `__coconut_cache__` will always be removed from `__file__`.
+
 #### `use_coconut_breakpoint`
 
 **coconut.api.use_coconut_breakpoint**(_on_=`True`)

coconut/api.py

@@ -34,9 +34,12 @@
 from coconut.command.util import proc_run_args
 from coconut.compiler import Compiler
 from coconut.constants import (
+    PY34,
     version_tag,
     code_exts,
     coconut_kernel_kwargs,
+    default_use_cache_dir,
+    coconut_cache_dir,
 )
 
 # -----------------------------------------------------------------------------------------------------------------------
@@ -182,18 +185,47 @@ class CoconutImporter(object):
     ext = code_exts[0]
     command = None
 
-    def __init__(self, *args):
+    def __init__(self, *args) -> None:
+        self.use_cache_dir(default_use_cache_dir)
         self.set_args(args)
 
+    def use_cache_dir(self, use_cache_dir):
+        """Set the cache directory if any to use for compiled Coconut files."""
+        if use_cache_dir:
+            if not PY34:
+                raise CoconutException("coconut.api.auto_compilation only supports the usage of a cache directory on Python 3.4+")
+            self.cache_dir = coconut_cache_dir
+        else:
+            self.cache_dir = None
+
     def set_args(self, args):
         """Set the Coconut command line args to use for auto compilation."""
         self.args = proc_run_args(args)
 
-    def run_compiler(self, path):
-        """Run the Coconut compiler on the given path."""
+    def cmd(self, *args):
+        """Run the Coconut compiler with the given args."""
         if self.command is None:
             self.command = Command()
-        return self.command.cmd([path] + self.args, interact=False)
+        return self.command.cmd(list(args) + self.args, interact=False)
+
+    def compile(self, path, package):
+        """Compile a path to a file or package."""
+        extra_args = []
+        if self.cache_dir:
+            if package:
+                cache_dir = os.path.join(path, self.cache_dir)
+            else:
+                cache_dir = os.path.join(os.path.dirname(path), self.cache_dir)
+            extra_args.append(cache_dir)
+        else:
+            cache_dir = None
+
+        if package:
+            self.cmd(path, *extra_args)
+            return cache_dir or path
+        else:
+            destpath, = self.cmd(path, *extra_args)
+            return destpath
 
     def find_coconut(self, fullname, path=None):
         """Searches for a Coconut file of the given name and compiles it."""
@@ -204,41 +236,47 @@ def find_coconut(self, fullname, path=None):
                 return None
             fullname = fullname[1:]
             basepaths.insert(0, path)
+
         path_tail = os.path.join(*fullname.split("."))
         for path_head in basepaths:
             path = os.path.join(path_head, path_tail)
             filepath = path + self.ext
-            dirpath = os.path.join(path, "__init__" + self.ext)
+            initpath = os.path.join(path, "__init__" + self.ext)
             if os.path.exists(filepath):
-                # Coconut file was found and compiled
-                destpath, = self.run_compiler(filepath)
-                return destpath
-            if os.path.exists(dirpath):
-                # Coconut package was found and compiled
-                self.run_compiler(path)
-                return path
+                return self.compile(filepath, package=False)
+            if os.path.exists(initpath):
+                return self.compile(path, package=True)
         return None
 
     def find_module(self, fullname, path=None):
         """Get a loader for a Coconut module if it exists."""
-        self.find_coconut(fullname, path)
-        # always return None to let Python import the compiled Coconut
-        return None
-
-    def find_spec(self, fullname, path, oldmodule=None):
+        destpath = self.find_coconut(fullname, path)
+        # return None to let Python do the import when nothing was found or compiling in-place
+        if destpath is None or not self.cache_dir:
+            return None
+        else:
+            from importlib.machinery import SourceFileLoader
+            return SourceFileLoader(fullname, destpath)
+
+    def find_spec(self, fullname, path=None, target=None):
         """Get a modulespec for a Coconut module if it exists."""
-        self.find_coconut(fullname, path)
-        # always return None to let Python import the compiled Coconut
-        return None
+        loader = self.find_module(fullname, path)
+        if loader is None:
+            return None
+        else:
+            from importlib.util import spec_from_loader
+            return spec_from_loader(fullname, loader)
 
 
 coconut_importer = CoconutImporter()
 
 
-def auto_compilation(on=True, args=None):
+def auto_compilation(on=True, args=None, use_cache_dir=None):
     """Turn automatic compilation of Coconut files on or off."""
     if args is not None:
         coconut_importer.set_args(args)
+    if use_cache_dir is not None:
+        coconut_importer.use_cache_dir(use_cache_dir)
     if on:
         if coconut_importer not in sys.meta_path:
             sys.meta_path.insert(0, coconut_importer)

coconut/api.pyi

@@ -103,7 +103,11 @@ def use_coconut_breakpoint(on: bool = True) -> None: ...
 coconut_importer: Any = ...
 
 
-def auto_compilation(on: bool = True, args: Iterable[Text] | None = None) -> None: ...
+def auto_compilation(
+    on: bool = True,
+    args: Iterable[Text] | None = None,
+    use_cache_dir: bool | None = None,
+) -> None: ...
 
 
 def get_coconut_encoding(encoding: Text = ...) -> Any: ...

coconut/command/command.py

@@ -69,6 +69,8 @@
     jupyter_console_commands,
     default_jobs,
     create_package_retries,
+    default_use_cache_dir,
+    coconut_cache_dir,
 )
 from coconut.util import (
     univ_open,
@@ -142,23 +144,35 @@ def start(self, run=False):
         if run:
             args, argv = [], []
             # for coconut-run, all args beyond the source file should be wrapped in an --argv
+            source = None
             for i in range(1, len(sys.argv)):
                 arg = sys.argv[i]
-                args.append(arg)
                 # if arg is source file, put everything else in argv
-                if not arg.startswith("-") and can_parse(arguments, args[:-1]):
+                if not arg.startswith("-") and can_parse(arguments, args):
+                    source = arg
                     argv = sys.argv[i + 1:]
                     break
+                else:
+                    args.append(arg)
             args = proc_run_args(args)
             if "--run" in args:
                 logger.warn("extraneous --run argument passed; coconut-run implies --run")
             else:
                 args.append("--run")
-            self.cmd(args, argv=argv)
+            dest = None
+            if source is not None:
+                source = fixpath(source)
+                args.append(source)
+                if default_use_cache_dir:
+                    if memoized_isfile(source):
+                        dest = os.path.join(os.path.dirname(source), coconut_cache_dir)
+                    else:
+                        dest = os.path.join(source, coconut_cache_dir)
+            self.cmd(args, argv=argv, use_dest=dest)
         else:
             self.cmd()
 
-    def cmd(self, args=None, argv=None, interact=True, default_target=None):
+    def cmd(self, args=None, argv=None, interact=True, default_target=None, use_dest=None):
         """Process command-line arguments."""
         with self.handling_exceptions():
             if args is None:
@@ -171,6 +185,9 @@ def cmd(self, args=None, argv=None, interact=True, default_target=None):
                 parsed_args.argv = argv
             if parsed_args.target is None:
                 parsed_args.target = default_target
+            if use_dest is not None and not parsed_args.no_write:
+                internal_assert(parsed_args.dest is None, "coconut-run got passed a dest", parsed_args)
+                parsed_args.dest = use_dest
             self.exit_code = 0
             self.stack_size = parsed_args.stack_size
             result = self.run_with_stack_size(self.execute_args, parsed_args, interact, original_args=args)

coconut/command/util.py

@@ -585,9 +585,8 @@ def build_vars(path=None, init=False):
             "__name__": "__main__",
             "__package__": None,
             "reload": reload,
+            "__file__": None if path is None else fixpath(path)
         }
-        if path is not None:
-            init_vars["__file__"] = fixpath(path)
         if init:
             # put reserved_vars in for auto-completion purposes only at the very beginning
             for var in reserved_vars:

coconut/compiler/header.py

@@ -38,6 +38,7 @@
     self_match_types,
     is_data_var,
     data_defaults_var,
+    coconut_cache_dir,
 )
 from coconut.util import (
     univ_open,
@@ -224,6 +225,7 @@ def process_header_args(which, use_hash, target, no_tco, strict, no_wrap):
         module_docstring='"""Built-in Coconut utilities."""\n\n' if which == "__coconut__" else "",
         __coconut__=make_py_str("__coconut__", target),
         _coconut_cached__coconut__=make_py_str("_coconut_cached__coconut__", target),
+        coconut_cache_dir=make_py_str(coconut_cache_dir, target),
         object="" if target.startswith("3") else "(object)",
         comma_object="" if target.startswith("3") else ", object",
         comma_slash=", /" if target_info >= (3, 8) else "",
@@ -841,19 +843,21 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap):
     elif target_info >= (3, 5):
         header += "from __future__ import generator_stop\n"
 
-    header += "import sys as _coconut_sys\n"
+    header += '''import sys as _coconut_sys
+import os as _coconut_os
+'''
 
     if which.startswith("package") or which == "__coconut__":
         header += "_coconut_header_info = " + header_info + "\n"
 
+    levels_up = None
     if which.startswith("package"):
         levels_up = int(assert_remove_prefix(which, "package:"))
         coconut_file_dir = "_coconut_os.path.dirname(_coconut_os.path.abspath(__file__))"
         for _ in range(levels_up):
             coconut_file_dir = "_coconut_os.path.dirname(" + coconut_file_dir + ")"
-        return header + prepare(
+        header += prepare(
             '''
-import os as _coconut_os
 _coconut_cached__coconut__ = _coconut_sys.modules.get({__coconut__})
 _coconut_file_dir = {coconut_file_dir}
 _coconut_pop_path = False
@@ -886,12 +890,37 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap):
         ).format(
             coconut_file_dir=coconut_file_dir,
             **format_dict
-        ) + section("Compiled Coconut")
+        )
 
     if which == "sys":
-        return header + '''from coconut.__coconut__ import *
+        header += '''from coconut.__coconut__ import *
 from coconut.__coconut__ import {underscore_imports}
-'''.format(**format_dict) + section("Compiled Coconut")
+'''.format(**format_dict)
+
+    # remove coconut_cache_dir from __file__ if it was put there by auto compilation
+    header += prepare(
+        '''
+try:
+    __file__ = _coconut_os.path.abspath(__file__) if __file__ else __file__
+except NameError:
+    pass
+else:
+    if __file__ and {coconut_cache_dir} in __file__:
+        _coconut_file_comps = []
+        while __file__:
+            __file__, _coconut_file_comp = _coconut_os.path.split(__file__)
+            if not _coconut_file_comp:
+                _coconut_file_comps.append(__file__)
+                break
+            if _coconut_file_comp != {coconut_cache_dir}:
+                _coconut_file_comps.append(_coconut_file_comp)
+        __file__ = _coconut_os.path.join(*reversed(_coconut_file_comps))
+        ''',
+        newline=True,
+    ).format(**format_dict)
+
+    if which in ("package", "sys"):
+        return header + section("Compiled Coconut")
 
     # __coconut__, code, file
 

coconut/constants.py

@@ -584,6 +584,9 @@ def get_bool_env_var(env_var, default=False):
 main_prompt = ">>> "
 more_prompt = "    "
 
+default_use_cache_dir = PY34
+coconut_cache_dir = "__coconut_cache__"
+
 mypy_path_env_var = "MYPYPATH"
 
 style_env_var = "COCONUT_STYLE"

coconut/root.py

@@ -26,7 +26,7 @@
 VERSION = "3.0.2"
 VERSION_NAME = None
 # False for release, int >= 1 for develop
-DEVELOP = 21
+DEVELOP = 22
 ALPHA = False  # for pre releases rather than post releases
 
 assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"