Merge #117

117: koch.py: support csources in release archives r=alaviss a=alaviss

This pull enables `koch.py` to build a bootstrap compiler from bundled csources in release archives.

This allow a build from the source archive to be network-less.

Included is a small fix to ensure that `koch.py` is packaged in release archives.

This pull also changes the source archive layout a fair bit, with all of `csources` being placed in `/build/csources` instead of `/`.

Co-authored-by: Leorize <leorize+oss@disroot.org>

compiler/installer.ini

@@ -50,7 +50,7 @@ Start: "doc/html/overview.html"
 
 [Other]
 Files: "copying.txt"
-Files: "koch.nim"
+Files: "koch.py"
 
 Files: "icons/nim.ico"
 Files: "icons/nim.rc"

koch.py

@@ -69,36 +69,55 @@ class Bootstrap:
     # The bootstrap compiler binary store location
     Binaries: ClassVar = Source / "bin"
 
-    # The location of
+    # The location of build configuration
     ConfigPath: ClassVar = NimSource / "config" / "build_config.txt"
 
+    # The location of bootstrap source release file
+    SourceReleasePath: ClassVar = Source / "csources-release"
+
     def __init__(self) -> None:
         """Load configuration and initialize the object"""
 
         self.sourceUrl = ""
         self.sourceCommit = ""
+        self.sourceRelease = ""
+
+        # The tag used for the versioned bootstrap compiler
+        versioned_bin_tag = ""
+
+        try:
+            with open(Bootstrap.SourceReleasePath) as releaseFile:
+                self.sourceRelease = releaseFile.read().strip()
+                versioned_bin_tag = self.sourceRelease
+        except OSError:
+            # There are no release csources bundled
+            pass
+
+        # Read build configuration if there are no release csources bundled
+        if self.sourceRelease == "":
+            with open(Bootstrap.ConfigPath) as config:
+                for line in config:
+                    (key, _, value) = line.rstrip().partition("=")
+
+                    if key == "nim_csourcesUrl":
+                        self.sourceUrl = value
+                    elif key == "nim_csourcesHash":
+                        self.sourceCommit = value
+
+            if self.sourceUrl == "":
+                raise RuntimeError(
+                    "Could not find the configuration for bootstrap source URL"
+                )
 
-        with open(Bootstrap.ConfigPath) as config:
-            for line in config:
-                (key, _, value) = line.rstrip().partition("=")
-
-                if key == "nim_csourcesUrl":
-                    self.sourceUrl = value
-                elif key == "nim_csourcesHash":
-                    self.sourceCommit = value
-
-        if self.sourceUrl == "":
-            raise RuntimeError(
-                "Could not find the configuration for bootstrap source URL"
-            )
+            if self.sourceCommit == "":
+                raise RuntimeError(
+                    "Could not find the configuration for bootstrap source commit"
+                )
 
-        if self.sourceCommit == "":
-            raise RuntimeError(
-                "Could not find the configuration for bootstrap source commit"
-            )
+            versioned_bin_tag = self.sourceCommit
 
         # The name of the compiler binary in the store
-        self.versioned_bin = exe(Bootstrap.Binaries / ("nim" + "-" + self.sourceCommit))
+        self.versioned_bin = exe(Bootstrap.Binaries / ("nim" + "-" + versioned_bin_tag))
 
     def is_built(self) -> bool:
         """Returns whether the bootstrap exist in the store"""
@@ -116,6 +135,10 @@ def _git(self, *args: Any) -> None:
 
     def fetch(self) -> None:
         """Download the bootstrap compiler source if needed"""
+        # There is no need to fetch if the release source is used
+        if self.sourceRelease != "":
+            return
+
         localSourceUrl = Bootstrap._git_capture(
             "remote", "get-url", "origin"
         ).stdout.rstrip()

tools/niminst/niminst.nim

@@ -21,6 +21,7 @@ const
   makeFile = "makefile"
   installShFile = "install.sh"
   deinstallShFile = "deinstall.sh"
+  csourcesReleaseFile = "csources-release"
 
 type
   AppType = enum appConsole, appGUI
@@ -651,20 +652,28 @@ proc xzDist(c: var ConfigData; windowsZip=false) =
          "./koch csource -d:danger.")
 
   if not windowsZip:
-    processFile(proj / buildBatFile, "build" / buildBatFile)
-    processFile(proj / buildBatFile32, "build" / buildBatFile32)
-    processFile(proj / buildBatFile64, "build" / buildBatFile64)
-    processFile(proj / buildShFile, "build" / buildShFile)
-    processFile(proj / makeFile, "build" / makeFile)
+    # Store csources in the location koch.py expects
+    let bootstrapDist = proj / "build" / "csources"
+    processFile(bootstrapDist / buildBatFile, "build" / buildBatFile)
+    processFile(bootstrapDist / buildBatFile32, "build" / buildBatFile32)
+    processFile(bootstrapDist / buildBatFile64, "build" / buildBatFile64)
+    processFile(bootstrapDist / buildShFile, "build" / buildShFile)
+    processFile(bootstrapDist / makeFile, "build" / makeFile)
+
+    # Tag the source so koch.py knows to use it instead of cloning a fresh copy
+    writeFile(tmpDir / csourcesReleaseFile, c.version)
+    processFile(bootstrapDist / csourcesReleaseFile, tmpDir / csourcesReleaseFile)
+
     processFile(proj / installShFile, installShFile)
     processFile(proj / deinstallShFile, deinstallShFile)
     template processFileAux(src, dst) = processFile(dst, src)
-    gatherFiles(processFileAux, c.libpath, proj / "c_code")
+    gatherFiles(processFileAux, c.libpath, bootstrapDist / "c_code")
     for osA in 1..c.oses.len:
       for cpuA in 1..c.cpus.len:
         var dir = buildDir(osA, cpuA)
         for k, f in walkDir("build" / dir):
-          if k == pcFile: processFile(proj / dir / extractFilename(f), f)
+          if k == pcFile:
+            processFile(bootstrapDist / dir / extractFilename(f), f)
   else:
     for f in items(c.cat[fcWinBin]):
       let filename = f.extractFilename