Merge pull request #1945 from tweag/use-json-for-bindists

Use a JSON file to keep GHC bindist information

.github/update-ghc.py

@@ -1,33 +1,19 @@
 #!/usr/bin/env python
 
 import os, sys, re
-import importlib
-import importlib.util
-import importlib.machinery
 import http.client
 import json
 from pathlib import PurePosixPath, Path
 from pprint import pprint
 from urllib.parse import quote
-from subprocess import Popen, PIPE, STDOUT, DEVNULL, check_output, run
+from subprocess import check_call, run
 
-def import_path(path):
-    module_name = os.path.basename(path).replace('-', '_')
-    spec = importlib.util.spec_from_loader(
-        module_name,
-        importlib.machinery.SourceFileLoader(module_name, path)
-    )
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    sys.modules[module_name] = module
-    return module
+GHC_GITLAB_SERVER = "gitlab.haskell.org"
 
 
-GHC_GITLAB_SERVER="gitlab.haskell.org"
-
 def get_gitlab_tags_with_prefix(project_id, prefix):
     connection = http.client.HTTPSConnection(GHC_GITLAB_SERVER)
-    encoded_project_id = encoded_project_id = quote(project_id, safe='')
+    encoded_project_id = encoded_project_id = quote(project_id, safe="")
     endpoint = f"/api/v4/projects/{encoded_project_id}/repository/tags?search=^{prefix}&order-by=version"
 
     connection.request("GET", endpoint)
@@ -41,74 +27,96 @@ def get_gitlab_tags_with_prefix(project_id, prefix):
         print(f"Error: {response.status} - {response.reason}")
         return None
 
+
 project_id = "ghc/ghc"
 
 # matches GHC release versions
 version_re = re.compile("ghc-(?P<version>.*)-release")
 
+
 def parse_version(s):
     return tuple(map(int, s.split(".")))
 
+
 def main():
     SCRIPT_PATH = Path(__file__)
 
     if len(sys.argv) != 2:
         sys.exit(f"usage: {SCRIPT_PATH.name} GHC_MAJOR_MINOR")
 
-    bindist = import_path(str(SCRIPT_PATH.parent.parent.joinpath('haskell/private/ghc_bindist_generated.bzl')))
-
     GHC_MAJOR_MINOR = sys.argv[1]
 
-    versions = [ version for version in bindist.GHC_BINDIST if version.startswith(GHC_MAJOR_MINOR) ]
-
     prefix = f"ghc-{GHC_MAJOR_MINOR}."
 
     tags = get_gitlab_tags_with_prefix(project_id, prefix)
 
-    if tags:
-        releases = [ m.group("version") for tag in tags if (m := version_re.match(tag["name"])) ]
+    if not tags:
+        print("no tags found for prefix", prefix, file=sys.stderr)
+        sys.exit(0)
+
+    with open(
+        SCRIPT_PATH.parent.parent.joinpath(
+            "haskell/private/ghc_bindist_generated.json"
+        ),
+        "r+",
+    ) as bindist_json:
+        bindists = json.load(bindist_json)
+
+        versions = [
+            version for version in bindists if version.startswith(GHC_MAJOR_MINOR)
+        ]
+
+        releases = [
+            m.group("version") for tag in tags if (m := version_re.match(tag["name"]))
+        ]
 
         latest_release = releases[0]
 
         if latest_release in versions:
             print(f"GHC {GHC_MAJOR_MINOR} is up-to-date", file=sys.stderr)
-        else:
-            print("found update:", latest_release, file=sys.stderr)
+            sys.exit(0)
 
-            replace = re.compile(r'^(?P<indent>\s+)\{\s*"version"\s*:\s*"' + re.escape(GHC_MAJOR_MINOR + '.'), re.MULTILINE)
+        print("found update:", latest_release, file=sys.stderr)
 
-            gen_script_path = SCRIPT_PATH.parent.parent.joinpath('haskell/gen_ghc_bindist.py')
-            with open(gen_script_path, 'r+') as gen:
-                gen_script = gen.read()
+        replace = re.compile(
+            r'^(?P<indent>\s+)\{\s*"version"\s*:\s*"'
+            + re.escape(GHC_MAJOR_MINOR + "."),
+            re.MULTILINE,
+        )
 
-                print(" 1. modify haskell/gen_ghc_bindist.py", file=sys.stderr)
+        print(" 1. modify haskell/gen_ghc_bindist.py", file=sys.stderr)
 
-                added_version = replace.sub(fr'''\g<indent>{{ "version": { repr(latest_release) },
+        gen_script_path = SCRIPT_PATH.parent.parent.joinpath(
+            "haskell/gen_ghc_bindist.py"
+        )
+        with open(gen_script_path, "r+") as gen:
+            gen_script = gen.read()
+
+            added_version = replace.sub(
+                rf"""\g<indent>{{ "version": { repr(latest_release) },
 \g<indent>  "ignore_suffixes": [".bz2", ".lz", ".zip"] }},
-\g<0>''', gen_script, count=1)
+\g<0>""",
+                gen_script,
+                count=1,
+            )
 
-                if added_version is gen_script:
-                    sys.exit(f"could not add new version {latest_release} using regex {replace}")
-                else:
-                    gen.truncate(0)
-                    gen.seek(0)
-                    gen.write(added_version)
+            if added_version is gen_script:
+                sys.exit(
+                    f"could not add new version {latest_release} using regex {replace}"
+                )
 
-            print(" 2. call haskell/gen_ghc_bindist.py", file=sys.stderr)
+            gen.truncate(0)
+            gen.seek(0)
+            gen.write(added_version)
 
-            bzl = check_output([sys.executable, 'haskell/gen_ghc_bindist.py'])
+        print(" 2. call haskell/gen_ghc_bindist.py", file=sys.stderr)
 
-            print(" 3. format bzl code, write to haskell/private/ghc_bindist_generated.bzl", file=sys.stderr)
-            with open('haskell/private/ghc_bindist_generated.bzl', 'r+') as generated:
-                run(['buildifier'], check=True, input=bzl, stdout=generated)
-                generated.truncate()
+        check_call([sys.executable, "haskell/gen_ghc_bindist.py"])
 
-            if 'GITHUB_OUTPUT' in os.environ:
-                with open(os.environ["GITHUB_OUTPUT"], 'a') as output:
-                    print(f"latest={ latest_release }", file=output)
-    else:
-        print("no tags found for prefix", prefix, file=sys.stderr)
+        if "GITHUB_OUTPUT" in os.environ:
+            with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+                print(f"latest={ latest_release }", file=output)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

.github/workflows/update-ghc.yaml

@@ -28,8 +28,8 @@ jobs:
         if: steps.ghc_update.outputs.latest != ''
         uses: peter-evans/create-pull-request@v5
         with:
-          commit-message: "[automation] Add GHC version "
-          title: "[update] GHC ${{ steps.ghc_update.latest }}"
+          commit-message: "Add GHC bindist version ${{ steps.ghc_update.outputs.latest }}"
+          title: "[update] GHC ${{ matrix.ghc }}"
           branch: "automation/update-ghc-${{ matrix.ghc }}"
           base: master
           token: "${{ secrets.PR_TRIGGER_TOKEN }}"

extensions/haskell_toolchains.bzl

@@ -2,6 +2,7 @@
 
 load(
     "@rules_haskell//haskell:ghc_bindist.bzl",
+    "bindist_info_for_version",
     "ghc_bindist",
     "ghc_bindist_toolchain_declaration",
     "ghc_bindists_toolchain_declarations",
@@ -42,7 +43,7 @@ _bindist_tag = tag_class(
         ),
         "target": attr.string(
             mandatory = True,
-            doc = "The desired architecture (See [ghc_bindist_generated.bzl](https://github.com/tweag/rules_haskell/blob/master/haskell/private/ghc_bindist_generated.bzl))",
+            doc = "The desired architecture (See [ghc_bindist_generated.json](https://github.com/tweag/rules_haskell/blob/master/haskell/private/ghc_bindist_generated.json))",
         ),
         "ghcopts": attr.string_list(
             doc = "[see rules_haskell_toolchains](toolchain.html#rules_haskell_toolchains-ghcopts)",
@@ -147,6 +148,9 @@ def _haskell_toolchains_impl(mctx):
             # ones would have the same constraints and lower priority.
             found_bindists = True
             bindists_tag = module.tags.bindists[0]
+
+            targets = bindist_info_for_version(mctx, bindists_tag.version).keys()
+
             haskell_register_ghc_bindists(
                 version = bindists_tag.version,
                 ghcopts = bindists_tag.ghcopts,
@@ -155,9 +159,10 @@ def _haskell_toolchains_impl(mctx):
                 cabalopts = bindists_tag.cabalopts,
                 locale = bindists_tag.locale,
                 register = False,
+                targets = targets,
             )
             toolchain_declarations.extend(
-                ghc_bindists_toolchain_declarations(bindists_tag.version),
+                ghc_bindists_toolchain_declarations(mctx, bindists_tag.version),
             )
 
     all_toolchains(

haskell/BUILD.bazel

@@ -146,7 +146,7 @@ py_binary(
     visibility = ["//visibility:public"],
 )
 
-# generate the _GHC_BINDISTS dict
+# generate the GHC bindist JSON file
 py_binary(
     name = "gen-ghc-bindist",
     srcs = [":gen_ghc_bindist.py"],

haskell/gen_ghc_bindist.py

@@ -4,7 +4,8 @@
 # download paths and hashes, for maintainers.
 # It uses the hashes provided by download.haskell.org.
 
-import pprint
+import os
+import json
 import sys
 from urllib.request import urlopen
 from distutils.version import StrictVersion
@@ -169,16 +170,12 @@ def select_one(xs, ys):
                 )
         ghc_bindists[ver] = arch_dists
 
-    # Print to stdout. Be aware that you can't `> foo.bzl`,
-    # because that truncates the source file which is needed
-    # for bazel to run in the first place.
-    print("""\
-# Generated with `bazel run @rules_haskell//haskell:gen-ghc-bindist`
-# To add a version or architecture, edit the constants in haskell/gen_ghc_bindist.py,
-# regenerate the dict and copy it here.
-GHC_BINDIST = \\""")
-    print("{")
-    for version in sorted(ghc_bindists.keys(), key=StrictVersion):
-       print('    ', repr(version), end=': ')
-       print(pprint.pformat(ghc_bindists[version], indent=8), end=',\n')
-    print("}")
+    ghc_versions = { version: ghc_bindists[version] for version in sorted(ghc_bindists.keys(), key=StrictVersion) }
+
+    working_directory = os.environ.get("BUILD_WORKING_DIRECTORY", ".")
+
+    with open(os.path.join(working_directory, "haskell/private/ghc_bindist_generated.json"), "w", encoding="utf-8") as json_file:
+        json.dump(ghc_versions, json_file, indent=4)
+        json_file.write('\n')
+
+

haskell/ghc_bindist.bzl

@@ -17,7 +17,6 @@ load(
     "resolve_labels",
 )
 load(":private/validate_attrs.bzl", "check_deprecated_attribute_usage")
-load(":private/ghc_bindist_generated.bzl", "GHC_BINDIST")
 
 # If you change this, change stackage's version in the start script
 # (see stackage.org).
@@ -43,6 +42,20 @@ def _split_version(version):
         fail("GHC version should be a triple: {}".format(version))
     return (int(vs[0]), int(vs[1]), int(vs[2]))
 
+def _load_bindists(mctx):
+    bindist_json = mctx.read(Label("@rules_haskell//haskell:private/ghc_bindist_generated.json"))
+    return json.decode(bindist_json)
+
+def bindist_info_for_version(ctx, version):
+    GHC_BINDIST = _load_bindists(ctx)
+
+    bindist = GHC_BINDIST.get(version or _GHC_DEFAULT_VERSION)
+
+    if bindist == None:
+        fail("Binary distribution of GHC {} not available.".format(version))
+
+    return bindist
+
 def _ghc_bindist_impl(ctx):
     filepaths = resolve_labels(ctx, [
         "@rules_haskell//haskell:ghc.BUILD.tpl",
@@ -53,10 +66,12 @@ def _ghc_bindist_impl(ctx):
     target = ctx.attr.target
     os, _, arch = target.partition("_")
 
-    if GHC_BINDIST[version].get(target) == None:
+    bindist = bindist_info_for_version(ctx, version)
+
+    if target not in bindist:
         fail("Operating system {0} does not have a bindist for GHC version {1}".format(ctx.os.name, ctx.attr.version))
     else:
-        url, sha256 = GHC_BINDIST[version][target]
+        url, sha256 = bindist[target]
 
     bindist_dir = ctx.path(".")  # repo path
 
@@ -259,7 +274,6 @@ _ghc_bindist = repository_rule(
     attrs = {
         "version": attr.string(
             default = _GHC_DEFAULT_VERSION,
-            values = GHC_BINDIST.keys(),
             doc = "The desired GHC version",
         ),
         "target": attr.string(),
@@ -373,17 +387,18 @@ _windows_cc_toolchain = repository_rule(
 
 # Toolchains declarations for bindists used by the `haskell_toolchains` module
 # extension to register all the haskell toolchains in the same BUILD file
-def ghc_bindists_toolchain_declarations(version):
+def ghc_bindists_toolchain_declarations(mctx, version):
     version = version or _GHC_DEFAULT_VERSION
-    if not GHC_BINDIST.get(version):
-        fail("Binary distribution of GHC {} not available.".format(version))
+
+    bindist = bindist_info_for_version(mctx, version)
+
     return [
         ghc_bindist_toolchain_declaration(
             target = target,
             bindist_name = "rules_haskell_ghc_{}".format(target),
             toolchain_name = "{}",
         )
-        for target in GHC_BINDIST[version]
+        for target in bindist
     ]
 
 def ghc_bindist(
@@ -494,6 +509,15 @@ def ghc_bindist(
         if register:
             native.register_toolchains("@{}//:windows_cc_toolchain".format(cc_toolchain_repo_name))
 
+
+_GHC_AVAILABLE_TARGETS = [
+    "darwin_amd64",
+    "darwin_arm64",
+    "linux_amd64",
+    "linux_arm64",
+    "windows_amd64",
+]
+
 def haskell_register_ghc_bindists(
         version = None,
         compiler_flags = None,
@@ -502,7 +526,8 @@ def haskell_register_ghc_bindists(
         repl_ghci_args = None,
         cabalopts = None,
         locale = None,
-        register = True):
+        register = True,
+        targets = _GHC_AVAILABLE_TARGETS):
     """ Register GHC binary distributions for all platforms as toolchains.
 
     See [rules_haskell_toolchains](toolchain.html#rules_haskell_toolchains).
@@ -516,11 +541,11 @@ def haskell_register_ghc_bindists(
       cabalopts: [see rules_haskell_toolchains](toolchain.html#rules_haskell_toolchains-cabalopts)
       locale: [see rules_haskell_toolchains](toolchain.html#rules_haskell_toolchains-locale)
       register: Whether to register the toolchains (must be set to False if bzlmod is enabled)
+      targets: A list of target platforms to generate bindists for, e.g. `["linux_amd64", "windows_amd64"]` (default: all)
     """
     version = version or _GHC_DEFAULT_VERSION
-    if not GHC_BINDIST.get(version):
-        fail("Binary distribution of GHC {} not available.".format(version))
-    for target in GHC_BINDIST[version]:
+
+    for target in targets:
         ghc_bindist(
             name = "rules_haskell_ghc_{}".format(target),
             target = target,