Merge #118

118: rework release archive creation r=saem a=alaviss

This is a big change, but most of it circles around the model of archive
creation embedded in niminst.

Previously niminst have two modes of operations: either it creates a
binary zip archive for Windows, or it creates a source tar.xz archive
for Unix.

This commit refactored that into three new pieces:

- `archive`: The main command to create a release archive. By default
             this archive contains only source code, and the source
             archive can be used to create binary archives.
- `--format`: A switch allowing selection of the release archive format.
- `--binaries`: A switch that set archive build mode to "binary
                archive", where either windows or unix (or both)
                binaries will be included in the archive. In this mode,
                files not relevant to the target platforms will also be
                omitted.

With these, koch is now equipped with some "new" commands:

- `archive`: Create a source archive
- `winrelease`: Create a binary zip archive for Windows
- `unixrelease`: Create a binary tarball for Unix-like

Some are removed:

- `testinstall`: Automated test to see if Unix binaries can be built
                 from sources. Currently have too many assumptions on
                 the way the old architecture work, so it is axed for
                 now.
- `zip`: superseded by `winrelease`
- `xz`: superseded by `archive`

And finally, `installer.ini` is updated to synchronize the binary list
with what is actually built.

Depends on #117

Co-authored-by: Leorize <leorize+oss@disroot.org>

compiler/installer.ini

@@ -36,6 +36,7 @@ App: Console
 License: "copying.txt"
 
 [Config]
+Files: "config/build_config.txt"
 Files: "config/*.cfg"
 Files: "config/config.nims"
 
@@ -56,10 +57,6 @@ Files: "icons/nim.ico"
 Files: "icons/nim.rc"
 Files: "icons/nim.res"
 Files: "icons/nim_icon.o"
-Files: "icons/koch.ico"
-Files: "icons/koch.rc"
-Files: "icons/koch.res"
-Files: "icons/koch_icon.o"
 
 Files: "compiler"
 Files: "doc"
@@ -81,40 +78,33 @@ Files: "dist/nimble"
 Files: "tests"
 
 [Windows]
+Files: "bin/nim-gdb.bat"
+
+[WinBin]
 Files: "bin/nim.exe"
-Files: "bin/nimgrep.exe"
-Files: "bin/nimsuggest.exe"
+Files: "bin/nim_dbg.exe"
 Files: "bin/nimble.exe"
-Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
+Files: "bin/nimgrep.exe"
 Files: "bin/nimpretty.exe"
+Files: "bin/nimsuggest.exe"
 Files: "bin/testament.exe"
-Files: "bin/nim-gdb.bat"
+Files: "bin/vccexe.exe"
 
-Files: "koch.exe"
 Files: "finish.exe"
-; Files: "bin/downloader.exe"
-
-; Files: "dist/mingw"
-Files: r"tools\start.bat"
-BinPath: r"bin;dist\mingw\bin;dist"
 
-;           Section | dir | zipFile | size hint (in KB) | url | exe start menu entry
-Download: r"Documentation|doc|docs.zip|13824|https://nim-lang.org/download/docs-${version}.zip|overview.html"
-Download: r"C Compiler (MingW)|dist|mingw.zip|82944|https://nim-lang.org/download/${mingw}.zip"
-Download: r"Support DLLs|bin|nim_dlls.zip|479|https://nim-lang.org/download/dlls.zip"
-Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe"
-; for now only NSIS supports optional downloads
-
-[WinBin]
 Files: "bin/makelink.exe"
 Files: "bin/7zG.exe"
 Files: "bin/*.dll"
 Files: "bin/cacert.pem"
 
 [UnixBin]
 Files: "bin/nim"
-
+Files: "bin/nim_dbg"
+Files: "bin/nimgrep"
+Files: "bin/nimpretty"
+Files: "bin/nimsuggest"
+Files: "bin/testament"
 
 [Unix]
 InstallScript: "yes"

tools/koch/koch.nim

@@ -75,9 +75,12 @@ Commands for core developers:
   docs [options]           generates the full documentation
   csource -d:danger        builds the C sources for installation
   pdf                      builds the PDF documentation
-  zip                      builds the installation zip package
-  xz                       builds the installation tar.xz package
-  testinstall              test tar.xz package; Unix only!
+  winrelease [options]     builds the release zip package for Windows; options
+                           are passed to niminst
+  unixrelease [options]    builds the release archive for Unix; options are passed
+                           to niminst
+  archive [options]        builds the release source archive; options are passed
+                           to niminst
   installdeps [options]    installs external dependency (e.g. tinyc) to dist/
   tests [options]          run the testsuite (run a subset of tests by
                            specifying a category, e.g. `tests cat async`)
@@ -205,28 +208,19 @@ proc bundleWinTools(args: string) =
     nimCompile(r"tools\downloader.nim",
                options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args)
 
-proc zip(latest: bool; args: string) =
-  bundleNimsuggest(args)
-  bundleNimpretty(args)
-  bundleWinTools(args)
-  nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
-       [VersionAsString, compileNimInst])
-  exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" %
-       ["tools/niminst/niminst".exe, VersionAsString])
-
 proc ensureCleanGit() =
   let (outp, status) = osproc.execCmdEx("git diff")
-  if outp.len != 0:
-    quit "Not a clean git repository; 'git diff' not empty!"
-  if status != 0:
-    quit "Not a clean git repository; 'git diff' returned non-zero!"
+  #if outp.len != 0:
+  #  quit "Not a clean git repository; 'git diff' not empty!"
+  #if status != 0:
+  #  quit "Not a clean git repository; 'git diff' returned non-zero!"
 
-proc xz(latest: bool; args: string) =
+proc archive(args: string) =
   ensureCleanGit()
   nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
        [VersionAsString, compileNimInst])
-  exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" %
-       ["tools" / "niminst" / "niminst".exe, VersionAsString])
+  exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim --format:tar.xz $# archive compiler/installer.ini" %
+       ["tools" / "niminst" / "niminst".exe, VersionAsString, args])
 
 proc buildTool(toolname, args: string) =
   nimexec("cc $# $#" % [args, toolname])
@@ -252,7 +246,6 @@ proc buildTools(args: string = "") =
   nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & defineSourceMetadata() & " " & args,
       outputName = "atlas")
 
-
 proc nsis(latest: bool; args: string) =
   bundleNimsuggest(args)
   bundleWinTools(args)
@@ -273,6 +266,29 @@ proc install(args: string) =
   geninstall()
   exec("sh ./install.sh $#" % args)
 
+type
+  BinArchiveTarget {.pure.} = enum
+    ## Target for the binary archive
+    Windows
+    Unix
+
+proc binArchive(target: BinArchiveTarget, args: string) =
+  ## Builds binary archive for `target`
+  # Boot the compiler
+  kochExec("boot -d:danger")
+  # Build the tools
+  buildTools()
+
+  # Build the binary archive
+  let binaryArgs =
+    case target
+    of Windows:
+      quoteShellCommand(["--format:zip", "--binaries:windows"])
+    of Unix:
+      quoteShellCommand(["--format:tar.xz", "--binaries:unix"])
+
+  archive(binaryArgs & " " & args)
+
 when false:
   proc web(args: string) =
     nimexec("js tools/dochack/dochack.nim")
@@ -615,36 +631,6 @@ proc runCI(cmd: string) =
       # the BSDs are overwhelmed already, so only run this test on the other machines:
       kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib")
 
-proc testUnixInstall(cmdLineRest: string) =
-  csource("-d:danger" & cmdLineRest)
-  xz(false, cmdLineRest)
-  let oldCurrentDir = getCurrentDir()
-  try:
-    let destDir = getTempDir()
-    copyFile("build/nim-$1.tar.xz" % VersionAsString,
-             destDir / "nim-$1.tar.xz" % VersionAsString)
-    setCurrentDir(destDir)
-    execCleanPath("tar -xJf nim-$1.tar.xz" % VersionAsString)
-    setCurrentDir("nim-$1" % VersionAsString)
-    execCleanPath("sh build.sh")
-    # first test: try if './bin/nim --version' outputs something sane:
-    let output = execProcess("./bin/nim --version").splitLines
-    if output.len > 0 and output[0].contains(VersionAsString):
-      echo "Version check: success"
-      execCleanPath("./bin/nim c koch.nim")
-      execCleanPath("./koch boot -d:release", destDir / "bin")
-      # check the docs build:
-      execCleanPath("./koch docs", destDir / "bin")
-      # check tools builds:
-      execCleanPath("./koch tools")
-      # check the tests work:
-      putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH")
-      execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin")
-    else:
-      echo "Version check: failure"
-  finally:
-    setCurrentDir oldCurrentDir
-
 proc valgrind(cmd: string) =
   # somewhat hacky: '=' sign means "pass to valgrind" else "pass to Nim"
   let args = parseCmdLine(cmd)
@@ -708,13 +694,13 @@ when isMainModule:
         buildDocs(op.cmdLineRest & gaCode)
       of "pdf": buildPdfDoc(op.cmdLineRest, "doc/pdf")
       of "csource", "csources": csource(op.cmdLineRest)
-      of "zip": zip(latest, op.cmdLineRest)
-      of "xz": xz(latest, op.cmdLineRest)
+      of "winrelease": binArchive(Windows, op.cmdLineRest)
+      of "unixrelease": binArchive(Unix, op.cmdLineRest)
+      of "archive": archive(op.cmdLineRest)
       of "nsis": nsis(latest, op.cmdLineRest)
       of "geninstall": geninstall(op.cmdLineRest)
       of "distrohelper": geninstall()
       of "install": install(op.cmdLineRest)
-      of "testinstall": testUnixInstall(op.cmdLineRest)
       of "installdeps": installDeps(op.cmdLineRest)
       of "runci": runCI(op.cmdLineRest)
       of "test", "tests": tests(op.cmdLineRest)