From 510e85dc7fdbcdf548ed4eff077ba7359443beb5 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Mon, 16 Sep 2024 16:40:00 +0200 Subject: [PATCH 01/17] Pass correct rpath origin setting to linker on Darwin Similar to D54346421 --- haskell/haskell_ghci.bzl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/haskell/haskell_ghci.bzl b/haskell/haskell_ghci.bzl index 775566012..a59fbd121 100644 --- a/haskell/haskell_ghci.bzl +++ b/haskell/haskell_ghci.bzl @@ -57,6 +57,10 @@ load( "with_unique_str_sonames", ) load("@prelude//linking:types.bzl", "Linkage") +load( + "@prelude//cxx:linker.bzl", + "get_rpath_origin", +) load( "@prelude//utils:graph_utils.bzl", "depth_first_traversal", @@ -304,7 +308,7 @@ def _build_haskell_omnibus_so(ctx: AnalysisContext) -> HaskellOmnibusData: soname = "libghci_dependencies.so" extra_ldflags = [ "-rpath", - "$ORIGIN/{}".format(so_symlinks_root_path), + "{}/{}".format(get_rpath_origin(linker_info.type), so_symlinks_root_path) ] link_result = cxx_link_shared_library( ctx, From 918cc09cc50b8842711097973f4edbbce9473286 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Mon, 16 Sep 2024 16:47:05 +0200 Subject: [PATCH 02/17] Add `cxx_merge_cpreprocessors_actions` function with `AnalysisActions` parameter This is going to be useful with new dynamic action rules. --- cxx/preprocessor.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cxx/preprocessor.bzl b/cxx/preprocessor.bzl index 570119a6d..24d59c613 100644 --- a/cxx/preprocessor.bzl +++ b/cxx/preprocessor.bzl @@ -171,11 +171,14 @@ def cxx_inherited_preprocessor_infos(first_order_deps: list[Dependency]) -> list return filter(None, [x.get(CPreprocessorInfo) for x in first_order_deps]) def cxx_merge_cpreprocessors(ctx: AnalysisContext, own: list[CPreprocessor], xs: list[CPreprocessorInfo]) -> CPreprocessorInfo: + return cxx_merge_cpreprocessors_actions(ctx.actions, own, xs) + +def cxx_merge_cpreprocessors_actions(actions: AnalysisActions, own: list[CPreprocessor], xs: list[CPreprocessorInfo]) -> CPreprocessorInfo: kwargs = {"children": [x.set for x in xs]} if own: kwargs["value"] = own return CPreprocessorInfo( - set = ctx.actions.tset(CPreprocessorTSet, **kwargs), + set = actions.tset(CPreprocessorTSet, **kwargs), ) def _format_include_arg(flag: str, path: cmd_args, compiler_type: str) -> list[cmd_args]: From 5c764605b528a6e9a70457721d3257e4385b9f14 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 08:57:29 +0200 Subject: [PATCH 03/17] Add script to generate target metadata for Haskell --- haskell/tools/generate_target_metadata.py | 273 ++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100755 haskell/tools/generate_target_metadata.py diff --git a/haskell/tools/generate_target_metadata.py b/haskell/tools/generate_target_metadata.py new file mode 100755 index 000000000..4913342c1 --- /dev/null +++ b/haskell/tools/generate_target_metadata.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 + +"""Helper script to generate relevant metadata about Haskell targets. + +* The mapping from module source file to actual module name. +* The intra-package module dependency graph. +* The cross-package module dependencies. +* Which modules require Template Haskell. + +Note, boot files will be represented by a `-boot` suffix in the module name. + +The result is a JSON object with the following fields: +* `th_modules`: List of modules that require Template Haskell. +* `module_mapping`: Mapping from source inferred module name to actual module name, if different. +* `module_graph`: Intra-package module dependencies, `dict[modname, list[modname]]`. +* `package_deps`": Cross-package module dependencies, `dict[modname, dict[pkgname, list[modname]]`. +""" + +import argparse +import sys +import json +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + fromfile_prefix_chars="@") + parser.add_argument( + "--output", + required=True, + type=argparse.FileType("w"), + help="Write package metadata to this file in JSON format.") + parser.add_argument( + "--ghc", + required=True, + type=str, + help="Path to the Haskell compiler GHC.") + parser.add_argument( + "--ghc-arg", + required=False, + type=str, + action="append", + help="GHC compiler argument to forward to `ghc -M`, including package flags.") + parser.add_argument( + "--source-prefix", + required=True, + type=str, + help="The path prefix to strip of module sources to extract module names.") + parser.add_argument( + "--source", + required=True, + type=str, + action="append", + help="Haskell module source files of the current package.") + parser.add_argument( + "--package", + required=False, + type=str, + action="append", + default=[], + help="Package dependencies formated as `NAME:PREFIX_PATH`.") + parser.add_argument( + "--bin-path", + type=Path, + action="append", + default=[], + help="Add given path to PATH.", + ) + args = parser.parse_args() + + result = obtain_target_metadata(args) + + json.dump(result, args.output, indent=4, default=json_default_handler) + + +def json_default_handler(o): + if isinstance(o, set): + return sorted(o) + raise TypeError(f'Object of type {o.__class__.__name__} is not JSON serializable') + + +def obtain_target_metadata(args): + paths = [str(binpath) for binpath in args.bin_path if binpath.is_dir()] + ghc_depends = run_ghc_depends(args.ghc, args.ghc_arg, args.source, paths) + th_modules = determine_th_modules(ghc_depends) + module_mapping = determine_module_mapping(ghc_depends, args.source_prefix) + module_graph = determine_module_graph(ghc_depends) + package_deps = determine_package_deps(ghc_depends) + return { + "th_modules": th_modules, + "module_mapping": module_mapping, + "module_graph": module_graph, + "package_deps": package_deps, + } + + +def load_toolchain_packages(filepath): + with open(filepath, "r") as f: + return json.load(f) + + +def determine_th_modules(ghc_depends): + return [ + modname + for modname, properties in ghc_depends.items() + if uses_th(properties.get("options", [])) + ] + + +__TH_EXTENSIONS = ["TemplateHaskell", "TemplateHaskellQuotes", "QuasiQuotes"] + + +def uses_th(opts): + """Determine if a Template Haskell extension is enabled.""" + return any([f"-X{ext}" in opts for ext in __TH_EXTENSIONS]) + + +def determine_module_mapping(ghc_depends, source_prefix): + result = {} + + for modname, properties in ghc_depends.items(): + sources = list(filter(is_haskell_src, properties.get("sources", []))) + + if len(sources) != 1: + raise RuntimeError(f"Expected exactly one Haskell source for module '{modname}' but got '{sources}'.") + + apparent_name = src_to_module_name(strip_prefix_(source_prefix, sources[0]).lstrip("/")) + + if apparent_name != modname: + result[apparent_name] = modname + + boot_properties = properties.get("boot", None) + if boot_properties != None: + boot_modname = modname + "-boot" + boot_sources = list(filter(is_haskell_boot, boot_properties.get("sources", []))) + + if len(boot_sources) != 1: + raise RuntimeError(f"Expected at most one Haskell boot file for module '{modname}' but got '{boot_sources}'.") + + boot_apparent_name = src_to_module_name(strip_prefix_(source_prefix, sources[0]).lstrip("/")) + "-boot" + + if boot_apparent_name != boot_modname: + result[boot_apparent_name] = boot_modname + + return result + + +def determine_module_graph(ghc_depends): + module_deps = {} + for modname, description in ghc_depends.items(): + module_deps[modname] = description.get("modules", []) + [ + dep + "-boot" + for dep in description.get("modules-boot", []) + ] + + boot_description = description.get("boot", None) + if boot_description != None: + module_deps[modname + "-boot"] = boot_description.get("modules", []) + [ + dep + "-boot" + for dep in boot_description.get("modules-boot", []) + ] + + return module_deps + + +def determine_package_deps(ghc_depends): + package_deps = {} + + for modname, description in ghc_depends.items(): + for pkgdep in description.get("packages", {}): + pkgname = pkgdep.get("name") + package_deps.setdefault(modname, {})[pkgname] = pkgdep.get("modules", []) + + boot_description = description.get("boot", None) + if boot_description != None: + for pkgdep in boot_description.get("packages", {}): + pkgname = pkgdep.get("name") + package_deps.setdefault(modname + "-boot", {})[pkgname] = pkgdep.get("modules", []) + + return package_deps + + +def run_ghc_depends(ghc, ghc_args, sources, aux_paths): + with tempfile.TemporaryDirectory() as dname: + json_fname = os.path.join(dname, "depends.json") + make_fname = os.path.join(dname, "depends.make") + haskell_sources = list(filter(is_haskell_src, sources)) + args = [ + ghc, "-M", "-include-pkg-deps", + # Note: `-outputdir '.'` removes the prefix of all targets: + # backend/src/Foo/Util. => Foo/Util. + "-outputdir", ".", + "-dep-json", json_fname, + "-dep-makefile", make_fname, + ] + ghc_args + haskell_sources + + env = os.environ.copy() + path = env.get("PATH", "") + env["PATH"] = os.pathsep.join([path] + aux_paths) + + res = subprocess.run(args, env=env, capture_output=True) + if res.returncode != 0: + # Write the GHC command on failure. + print(shlex.join(args), file=sys.stderr) + + # Always forward stdout/stderr. + # Note, Buck2 swallows stdout on successful builds. + # Redirect to stderr to avoid this. + sys.stderr.buffer.write(res.stdout) + sys.stderr.buffer.write(res.stderr) + + if res.returncode != 0: + # Fail if GHC failed. + sys.exit(res.returncode) + + with open(json_fname) as f: + return json.load(f) + + +def src_to_module_name(x): + base, _ = os.path.splitext(x) + return base.replace("/", ".") + + +def is_haskell_src(x): + _, ext = os.path.splitext(x) + return ext in HASKELL_EXTENSIONS + + +def is_haskell_boot(x): + _, ext = os.path.splitext(x) + return ext in HASKELL_BOOT_EXTENSIONS + + +HASKELL_EXTENSIONS = [ + ".hs", + ".lhs", + ".hsc", + ".chs", + ".x", + ".y", +] + + +HASKELL_BOOT_EXTENSIONS = [ + ".hs-boot", + ".lhs-boot", +] + + +def strip_prefix_(prefix, s): + stripped = strip_prefix(prefix, s) + + if stripped == None: + return s + + return stripped + + +def strip_prefix(prefix, s): + if s.startswith(prefix): + return s[len(prefix):] + + return None + + +if __name__ == "__main__": + main() From 1c0f3c83d967a3057eba8c8ecac59855199733de Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 08:58:55 +0200 Subject: [PATCH 04/17] Add ghc wrapper script which writes ABI hash and a deps file --- haskell/tools/ghc_wrapper.py | 142 +++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 haskell/tools/ghc_wrapper.py diff --git a/haskell/tools/ghc_wrapper.py b/haskell/tools/ghc_wrapper.py new file mode 100755 index 000000000..0a1c20f2d --- /dev/null +++ b/haskell/tools/ghc_wrapper.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +"""Wrapper script to call ghc. + +It accepts a dep file where all used inputs are written to. For any passed ABI +hash file, the corresponding interface is marked as unused, so these can change +without triggering compilation actions. + +""" + +import argparse +import os +from pathlib import Path +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, add_help=False, fromfile_prefix_chars="@" + ) + parser.add_argument( + "--buck2-dep", + required=True, + help="Path to the dep file.", + ) + parser.add_argument( + "--buck2-packagedb-dep", + required=True, + help="Path to the dep file.", + ) + parser.add_argument( + "--buck2-package-db", + required=False, + nargs="*", + default=[], + help="Path to a package db that is used during the module compilation", + ) + parser.add_argument( + "--ghc", required=True, type=str, help="Path to the Haskell compiler GHC." + ) + parser.add_argument( + "--abi-out", + required=True, + type=Path, + help="Output path of the abi file to create.", + ) + parser.add_argument( + "--bin-path", + type=Path, + action="append", + default=[], + help="Add given path to PATH.", + ) + parser.add_argument( + "--bin-exe", + type=Path, + action="append", + default=[], + help="Add given exe (more specific than bin-path)", + ) + parser.add_argument( + "--extra-env-key", + type=str, + action="append", + default=[], + help="Extra environment variable name", + ) + parser.add_argument( + "--extra-env-value", + type=str, + action="append", + default=[], + help="Extra environment variable value", + ) + + args, ghc_args = parser.parse_known_args() + + cmd = [args.ghc] + ghc_args + + aux_paths = [str(binpath) for binpath in args.bin_path if binpath.is_dir()] + [str(os.path.dirname(binexepath)) for binexepath in args.bin_exe] + env = os.environ.copy() + path = env.get("PATH", "") + env["PATH"] = os.pathsep.join([path] + aux_paths) + + extra_env_keys = [str(k) for k in args.extra_env_key] + extra_env_values = [str(v) for v in args.extra_env_value] + assert len(extra_env_keys) == len(extra_env_values), "number of --extra-env-key and --extra-env-value flags must match" + n_extra_env = len(extra_env_keys) + if n_extra_env > 0: + for i in range(0, n_extra_env): + k = extra_env_keys[i] + v = extra_env_values[i] + env[k] = v + + # Note, Buck2 swallows stdout on successful builds. + # Redirect to stderr to avoid this. + returncode = subprocess.call(cmd, env=env, stdout=sys.stderr.buffer) + if returncode != 0: + return returncode + + recompute_abi_hash(args.ghc, args.abi_out) + + # write an empty dep file, to signal that all tagged files are unused + try: + with open(args.buck2_dep, "w") as f: + f.write("\n") + + except Exception as e: + # remove incomplete dep file + os.remove(args.buck2_dep) + raise e + + # write an empty dep file, to signal that all tagged files are unused + try: + with open(args.buck2_packagedb_dep, "w") as f: + for db in args.buck2_package_db: + f.write(db + "\n") + if not args.buck2_package_db: + f.write("\n") + + except Exception as e: + # remove incomplete dep file + os.remove(args.buck2_packagedb_dep) + raise e + + return 0 + + +def recompute_abi_hash(ghc, abi_out): + """Call ghc on the hi file and write the ABI hash to abi_out.""" + hi_file = abi_out.with_suffix("") + + cmd = [ghc, "-v0", "-package-env=-", "--show-iface-abi-hash", hi_file] + + hash = subprocess.check_output(cmd, text=True).split(maxsplit=1)[0] + + abi_out.write_text(hash) + + +if __name__ == "__main__": + main() From 5c073fe423e4c3f74b59c7a8a38301e03e23f947 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 09:00:05 +0200 Subject: [PATCH 05/17] Add `is_haskell_boot` helper function --- haskell/util.bzl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/haskell/util.bzl b/haskell/util.bzl index 80584cd3b..e3662ba58 100644 --- a/haskell/util.bzl +++ b/haskell/util.bzl @@ -41,6 +41,11 @@ HASKELL_EXTENSIONS = [ ".y", ] +HASKELL_BOOT_EXTENSIONS = [ + ".hs-boot", + ".lhs-boot", +] + # We take a named_set for srcs, which is sometimes a list, sometimes a dict. # In future we should only accept a list, but for now, cope with both. def srcs_to_pairs(srcs) -> list[(str, Artifact)]: @@ -53,6 +58,10 @@ def is_haskell_src(x: str) -> bool: _, ext = paths.split_extension(x) return ext in HASKELL_EXTENSIONS +def is_haskell_boot(x: str) -> bool: + _, ext = paths.split_extension(x) + return ext in HASKELL_BOOT_EXTENSIONS + def src_to_module_name(x: str) -> str: base, _ext = paths.split_extension(x) return base.replace("/", ".") From 04bc389857d9858b7233c982caba2d251b4f68ea Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 09:16:20 +0200 Subject: [PATCH 06/17] Add `HaskellToolchainLibrary` provider This is used to track libraries that are provided by the haskell toolchain (e.g. by Nix) in the `deps` attribute of Haskell rules. --- haskell/toolchain.bzl | 6 ++++++ haskell/util.bzl | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index f6c072fbf..fbd140ff7 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -39,3 +39,9 @@ HaskellToolchainInfo = provider( "script_template_processor": provider_field(typing.Any, default = None), }, ) + +HaskellToolchainLibrary = provider( + fields = { + "name": provider_field(str), + }, +) diff --git a/haskell/util.bzl b/haskell/util.bzl index e3662ba58..4ce4745a6 100644 --- a/haskell/util.bzl +++ b/haskell/util.bzl @@ -10,6 +10,10 @@ load( "@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo", ) +load( + "@prelude//haskell:toolchain.bzl", + "HaskellToolchainLibrary", +) load( "@prelude//haskell:library_info.bzl", "HaskellLibraryInfo", @@ -82,6 +86,15 @@ def attr_deps_haskell_link_infos(ctx: AnalysisContext) -> list[HaskellLinkInfo]: ], )) +def attr_deps_haskell_toolchain_libraries(ctx: AnalysisContext) -> list[HaskellToolchainLibrary]: + return filter( + None, + [ + d.get(HaskellToolchainLibrary) + for d in attr_deps(ctx) + ctx.attrs.template_deps + ], + ) + # DONT CALL THIS FUNCTION, you want attr_deps_haskell_link_infos instead def attr_deps_haskell_link_infos_sans_template_deps(ctx: AnalysisContext) -> list[HaskellLinkInfo]: return dedupe(filter( From 28821ff940e328f48f06304f6b7fa370a54a68dd Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:07:38 +0200 Subject: [PATCH 07/17] Add `haskell_toolchain_library` rule --- decls/haskell_rules.bzl | 11 +++++++++++ haskell/haskell.bzl | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/decls/haskell_rules.bzl b/decls/haskell_rules.bzl index 0233dcdd4..ca90d8773 100644 --- a/decls/haskell_rules.bzl +++ b/decls/haskell_rules.bzl @@ -184,6 +184,16 @@ haskell_library = prelude_rule( ), ) +haskell_toolchain_library = prelude_rule( + name = "haskell_toolchain_library", + docs = """ + Declare a library available as part of the GHC toolchain. + """, + attrs = { + }, +) + + haskell_prebuilt_library = prelude_rule( name = "haskell_prebuilt_library", docs = """ @@ -257,4 +267,5 @@ haskell_rules = struct( haskell_ide = haskell_ide, haskell_library = haskell_library, haskell_prebuilt_library = haskell_prebuilt_library, + haskell_toolchain_library = haskell_toolchain_library, ) diff --git a/haskell/haskell.bzl b/haskell/haskell.bzl index 320b8f936..975ea03ad 100644 --- a/haskell/haskell.bzl +++ b/haskell/haskell.bzl @@ -168,6 +168,11 @@ def _attr_preferred_linkage(ctx: AnalysisContext) -> Linkage: # -- +def haskell_toolchain_library_impl(ctx: AnalysisContext): + return [DefaultInfo(), HaskellToolchainLibrary(name = ctx.attrs.name)] + +# -- + def _get_haskell_prebuilt_libs( ctx, link_style: LinkStyle, From 15df14c25ab69c3a72c1d051c4caa34d387518bf Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 09:17:55 +0200 Subject: [PATCH 08/17] Add `generate_target_metadata` and `ghc_wrapper` tool targets --- haskell/tools/BUCK.v2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/haskell/tools/BUCK.v2 b/haskell/tools/BUCK.v2 index 3029719fc..e57116335 100644 --- a/haskell/tools/BUCK.v2 +++ b/haskell/tools/BUCK.v2 @@ -11,3 +11,15 @@ prelude.python_bootstrap_binary( main = "script_template_processor.py", visibility = ["PUBLIC"], ) + +prelude.python_bootstrap_binary( + name = "generate_target_metadata", + main = "generate_target_metadata.py", + visibility = ["PUBLIC"], +) + +prelude.python_bootstrap_binary( + name = "ghc_wrapper", + main = "ghc_wrapper.py", + visibility = ["PUBLIC"], +) From cf6e340e54a2a35e045bc0c7f20a7d4171785a27 Mon Sep 17 00:00:00 2001 From: Ian-Woo Kim Date: Thu, 29 Aug 2024 19:47:49 -0700 Subject: [PATCH 09/17] make compiler_flags be of arg type (#34) compiler_flags often needs macro like $(location :target). --- decls/haskell_common.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decls/haskell_common.bzl b/decls/haskell_common.bzl index 8a8ee90ec..374ddb0a1 100644 --- a/decls/haskell_common.bzl +++ b/decls/haskell_common.bzl @@ -28,7 +28,7 @@ def _deps_arg(): def _compiler_flags_arg(): return { - "compiler_flags": attrs.list(attrs.string(), default = [], doc = """ + "compiler_flags": attrs.list(attrs.arg(), default = [], doc = """ Flags to pass to the Haskell compiler when compiling this rule's sources. """), } From a43ba22a37523d1704a772fd597ebca5b8fb077f Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 17 Sep 2024 10:42:52 +0200 Subject: [PATCH 10/17] Add scripts to haskell rules --- decls/haskell_common.bzl | 13 +++++++++++++ decls/haskell_rules.bzl | 2 ++ 2 files changed, 15 insertions(+) diff --git a/decls/haskell_common.bzl b/decls/haskell_common.bzl index 374ddb0a1..29f33111c 100644 --- a/decls/haskell_common.bzl +++ b/decls/haskell_common.bzl @@ -40,9 +40,22 @@ def _exported_linker_flags_arg(): """), } +def _scripts_arg(): + return { + "_generate_target_metadata": attrs.dep( + providers = [RunInfo], + default = "prelude//haskell/tools:generate_target_metadata", + ), + "_ghc_wrapper": attrs.dep( + providers = [RunInfo], + default = "prelude//haskell/tools:ghc_wrapper", + ), + } + haskell_common = struct( srcs_arg = _srcs_arg, deps_arg = _deps_arg, compiler_flags_arg = _compiler_flags_arg, exported_linker_flags_arg = _exported_linker_flags_arg, + scripts_arg = _scripts_arg, ) diff --git a/decls/haskell_rules.bzl b/decls/haskell_rules.bzl index ca90d8773..b4ed4907f 100644 --- a/decls/haskell_rules.bzl +++ b/decls/haskell_rules.bzl @@ -48,6 +48,7 @@ haskell_binary = prelude_rule( haskell_common.srcs_arg() | haskell_common.compiler_flags_arg() | haskell_common.deps_arg() | + haskell_common.scripts_arg() | buck.platform_deps_arg() | { "contacts": attrs.list(attrs.string(), default = []), @@ -165,6 +166,7 @@ haskell_library = prelude_rule( haskell_common.srcs_arg() | haskell_common.compiler_flags_arg() | haskell_common.deps_arg() | + haskell_common.scripts_arg() | buck.platform_deps_arg() | native_common.link_whole(link_whole_type = attrs.bool(default = False)) | native_common.preferred_linkage(preferred_linkage_type = attrs.enum(Linkage.values())) | From 9e563a90642afafab5d8faaa96df857d46df368a Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Thu, 13 Jun 2024 13:26:11 +0200 Subject: [PATCH 11/17] Allow to declare dependencies for haskell sources manually --- decls/haskell_common.bzl | 3 +++ haskell/compile.bzl | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/decls/haskell_common.bzl b/decls/haskell_common.bzl index 29f33111c..480533170 100644 --- a/decls/haskell_common.bzl +++ b/decls/haskell_common.bzl @@ -24,6 +24,9 @@ def _deps_arg(): from which this rules sources import modules or native linkable rules exporting symbols this rules sources call into. """), + "srcs_deps": attrs.dict(attrs.source(), attrs.list(attrs.source()), default = {}, doc = """ + Allows to declare dependencies for sources manually, additionally to the dependencies automatically detected. + """), } def _compiler_flags_arg(): diff --git a/haskell/compile.bzl b/haskell/compile.bzl index 222838398..b70949223 100644 --- a/haskell/compile.bzl +++ b/haskell/compile.bzl @@ -198,6 +198,11 @@ def compile_args( arg_srcs = [] hidden_srcs = [] + + aux_deps = ctx.attrs.srcs_deps.get(module.source) + if aux_deps: + hidden_srcs(aux_deps) + for (path, src) in srcs_to_pairs(ctx.attrs.srcs): # hs-boot files aren't expected to be an argument to compiler but does need # to be included in the directory of the associated src file From 0175c444ba4720783e1339993bdc042a754eaf58 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:02:58 +0200 Subject: [PATCH 12/17] Add `get_source_prefixes` helper function --- haskell/util.bzl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/haskell/util.bzl b/haskell/util.bzl index 4ce4745a6..37c261cea 100644 --- a/haskell/util.bzl +++ b/haskell/util.bzl @@ -171,3 +171,34 @@ def get_artifact_suffix(link_style: LinkStyle, enable_profiling: bool, suffix: s if enable_profiling: artifact_suffix += "-prof" return artifact_suffix + suffix + +def _source_prefix(source: Artifact, module_name: str) -> str: + """Determine the directory prefix of the given artifact, considering that ghc has determined `module_name` for that file.""" + source_path = paths.replace_extension(source.short_path, "") + + module_name_for_file = src_to_module_name(source_path) + + # assert that source_path (without extension) and its module name have the same length + if len(source_path) != len(module_name_for_file): + fail("{} should have the same length as {}".format(source_path, module_name_for_file)) + + if module_name != module_name_for_file and module_name_for_file.endswith("." + module_name): + # N.B. the prefix could have some '.' characters in it, use the source_path to determine the prefix + return source_path[0:-len(module_name) - 1] + + return "" + + +def get_source_prefixes(srcs: list[Artifact], module_map: dict[str, str]) -> list[str]: + """Determine source prefixes for the given haskell files and a mapping from source file module name to module name.""" + source_prefixes = {} + for path, src in srcs_to_pairs(srcs): + if not is_haskell_src(path): + continue + + name = src_to_module_name(path) + real_name = module_map.get(name) + prefix = _source_prefix(src, real_name) if real_name else "" + source_prefixes[prefix] = None + + return source_prefixes.keys() From 8915ca00bb5410ebeaef12599d41baac8103fbc6 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:19:52 +0200 Subject: [PATCH 13/17] Make `ghci_lib_path` optional in template processor script --- haskell/tools/script_template_processor.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/haskell/tools/script_template_processor.py b/haskell/tools/script_template_processor.py index a105c4dbf..bd81cf626 100644 --- a/haskell/tools/script_template_processor.py +++ b/haskell/tools/script_template_processor.py @@ -68,16 +68,24 @@ def _replace_template_values( # user_ghci_path has to be handled separately because it needs to be passed # with the ghci_lib_path as the `-B` argument. - ghci_lib_canonical_path = os.path.realpath( - rel_toolchain_paths["ghci_lib_path"], - ) if user_ghci_path is not None: - script_template = re.sub( - pattern="", - repl="${{DIR}}/{user_ghci_path} -B{ghci_lib_path}".format( + ghci_lib_path = rel_toolchain_paths["ghci_lib_path"] + + if ghci_lib_path: + ghci_lib_canonical_path = os.path.realpath(ghci_lib_path) + + replacement="${{DIR}}/{user_ghci_path} -B{ghci_lib_path}".format( user_ghci_path=user_ghci_path, ghci_lib_path=ghci_lib_canonical_path, - ), + ) + else: + replacement="${{DIR}}/{user_ghci_path}".format( + user_ghci_path=user_ghci_path, + ) + + script_template = re.sub( + pattern="", + repl=replacement, string=script_template, ) From 92486a8351fecf6ba4a3c7fc24695781908c89f6 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:25:37 +0200 Subject: [PATCH 14/17] Add attributes to haskell rules - `external_tools`: pass executables called from Haskell compiler during preprocessing or compilation - `srcs_envs`: pass individual run-time env for each source compilation - `use_argsfile_at_link`: use response file at linking --- decls/haskell_common.bzl | 24 ++++++++++++++++++++++++ decls/haskell_rules.bzl | 6 ++++++ 2 files changed, 30 insertions(+) diff --git a/decls/haskell_common.bzl b/decls/haskell_common.bzl index 480533170..0f704f71d 100644 --- a/decls/haskell_common.bzl +++ b/decls/haskell_common.bzl @@ -55,10 +55,34 @@ def _scripts_arg(): ), } +def _external_tools_arg(): + return { + "external_tools": attrs.list(attrs.dep(providers = [RunInfo]), default = [], doc = """ + External executables called from Haskell compiler during preprocessing or compilation. +"""), + } + +def _srcs_envs_arg(): + return { + "srcs_envs": attrs.dict(attrs.source(), attrs.dict(attrs.string(), attrs.arg()), default = {}, doc = """ + Individual run-time env for each source compilation. +"""), + } + +def _use_argsfile_at_link_arg(): + return { + "use_argsfile_at_link": attrs.bool(default = False, doc = """ + Use response file at linking. +"""), + } + haskell_common = struct( srcs_arg = _srcs_arg, deps_arg = _deps_arg, compiler_flags_arg = _compiler_flags_arg, exported_linker_flags_arg = _exported_linker_flags_arg, scripts_arg = _scripts_arg, + external_tools_arg = _external_tools_arg, + srcs_envs_arg = _srcs_envs_arg, + use_argsfile_at_link_arg = _use_argsfile_at_link_arg, ) diff --git a/decls/haskell_rules.bzl b/decls/haskell_rules.bzl index b4ed4907f..61d2107ca 100644 --- a/decls/haskell_rules.bzl +++ b/decls/haskell_rules.bzl @@ -46,6 +46,9 @@ haskell_binary = prelude_rule( native_common.link_group_public_deps_label() | native_common.link_style() | haskell_common.srcs_arg() | + haskell_common.external_tools_arg() | + haskell_common.srcs_envs_arg () | + haskell_common.use_argsfile_at_link_arg () | haskell_common.compiler_flags_arg() | haskell_common.deps_arg() | haskell_common.scripts_arg() | @@ -164,6 +167,9 @@ haskell_library = prelude_rule( attrs = ( # @unsorted-dict-items haskell_common.srcs_arg() | + haskell_common.external_tools_arg() | + haskell_common.srcs_envs_arg() | + haskell_common.use_argsfile_at_link_arg() | haskell_common.compiler_flags_arg() | haskell_common.deps_arg() | haskell_common.scripts_arg() | From d9af4ce9a4245c35d7b0757219dfcd76ac02717c Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:32:11 +0200 Subject: [PATCH 15/17] Support HaskellToolchainLibs for ide BXL --- haskell/ide/ide.bxl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/haskell/ide/ide.bxl b/haskell/ide/ide.bxl index 57fcd4c05..a68eaaffa 100644 --- a/haskell/ide/ide.bxl +++ b/haskell/ide/ide.bxl @@ -7,7 +7,8 @@ load("@prelude//haskell:library_info.bzl", "HaskellLibraryProvider") load("@prelude//haskell:link_info.bzl", "HaskellLinkInfo") -load("@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo") +load("@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", "HaskellToolchainLibrary") +load("@prelude//haskell:util.bzl", "is_haskell_src", "srcs_to_pairs") load("@prelude//linking:link_info.bzl", "LinkStyle") load("@prelude//paths.bzl", "paths") @@ -142,12 +143,19 @@ def _solution_for_haskell_lib(ctx, target, exclude): hli = ctx.analysis(target).providers().get(HaskellLibraryProvider) haskellLibs = {} + toolchain_libs = [] + for dep in resolved_attrs.deps + resolved_attrs.template_deps: if exclude.get(dep.label) == None: providers = ctx.analysis(dep.label).providers() lb = providers.get(HaskellLinkInfo) if lb != None: haskellLibs[dep.label] = lb + continue + + lb = providers.get(HaskellToolchainLibrary) + if lb != None: + toolchain_libs.append(lb.name) sources = [] for item in ctx.output.ensure_multiple(resolved_attrs.srcs.values()): @@ -193,6 +201,7 @@ def _solution_for_haskell_lib(ctx, target, exclude): "flags": flags, "generated_dependencies": externalSourcesForTarget(ctx, target), "haskell_deps": haskellLibs, + "toolchain_libs": toolchain_libs, "import_dirs": import_dirs.keys(), "sources": sources, "targets": targetsForTarget(ctx, target), @@ -269,6 +278,9 @@ def _assembleSolution(ctx, linkStyle, result): ctx.output.ensure_multiple(hli.libs) ctx.output.ensure_multiple(hli.import_dirs.values()) package_dbs[hli.db] = () + for lib in result["toolchain_libs"]: + flags.append("-package") + flags.append(lib) for pkgdb in ctx.output.ensure_multiple(package_dbs.keys()): flags.append("-package-db") flags.append(pkgdb.abs_path()) From f1b87533bc32583f04fa2e36ffae612f7276dda1 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:34:22 +0200 Subject: [PATCH 16/17] Make cc toolchain tools optional --- haskell/ide/ide.bxl | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/haskell/ide/ide.bxl b/haskell/ide/ide.bxl index a68eaaffa..104238caf 100644 --- a/haskell/ide/ide.bxl +++ b/haskell/ide/ide.bxl @@ -173,11 +173,6 @@ def _solution_for_haskell_lib(ctx, target, exclude): haskell_toolchain = ctx.analysis(resolved_attrs._haskell_toolchain.label) toolchain = haskell_toolchain.providers().get(HaskellToolchainInfo) - binutils_path = paths.join(root, toolchain.ghci_binutils_path) - cc_path = paths.join(root, toolchain.ghci_cc_path) - cxx_path = paths.join(root, toolchain.ghci_cxx_path) - cpp_path = paths.join(root, toolchain.ghci_cpp_path) - flags = [ "-this-unit-id", "fbcode_fake_unit_id", @@ -187,13 +182,31 @@ def _solution_for_haskell_lib(ctx, target, exclude): "-no-global-package-db", "-no-user-package-db", "-hide-all-packages", - "-pgma%s" % cc_path, - "-pgml%s" % cxx_path, - "-pgmc%s" % cc_path, - "-pgmP%s" % cpp_path, - "-opta-B%s" % binutils_path, - "-optc-B%s" % binutils_path, + "-i", ] + + if toolchain.ghci_binutils_path: + binutils_path = paths.join(root, toolchain.ghci_binutils_path) + flags.extend([ + "-opta-B%s" % binutils_path, + "-optc-B%s" % binutils_path, + ]) + + if toolchain.ghci_cc_path: + cc_path = paths.join(root, toolchain.ghci_cc_path) + flags.extend([ + "-pgma%s" % cc_path, + "-pgmc%s" % cc_path, + ]) + + if toolchain.ghci_cxx_path: + cxx_path = paths.join(root, toolchain.ghci_cxx_path) + flags.append("-pgml%s" % cxx_path) + + if toolchain.ghci_cpp_path: + cpp_path = paths.join(root, toolchain.ghci_cpp_path) + flags.append("-pgmP%s" % cpp_path) + flags.extend(resolved_attrs.compiler_flags) return { From bb7b5572eddaf6366d20a30b297c13383b08b461 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Tue, 17 Sep 2024 11:57:18 +0200 Subject: [PATCH 17/17] Determine haskell dependencies and compile modules according to the dependency graph This commit comprises all of the changes needed to use meta data from the Haskell compiler (a JSON file gathered with `ghc -M`) and use that inside of dynamic actions to create module compilation actions, including tracking cross-package dependencies and handling template haskell usage accordingly. This also allows us to put sources into different sub-directories. Co-authored-by: Ian-Woo Kim Co-authored-by: Andreas Herrmann --- haskell/compile.bzl | 861 +++++++++++++++++++++++++++++++----- haskell/haskell.bzl | 572 ++++++++++++++++++------ haskell/haskell_ghci.bzl | 8 +- haskell/haskell_haddock.bzl | 229 +++++++--- haskell/library_info.bzl | 40 +- haskell/toolchain.bzl | 25 ++ 6 files changed, 1425 insertions(+), 310 deletions(-) diff --git a/haskell/compile.bzl b/haskell/compile.bzl index b70949223..015912cc2 100644 --- a/haskell/compile.bzl +++ b/haskell/compile.bzl @@ -5,26 +5,45 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//utils:arglike.bzl", "ArgLike") + load( "@prelude//cxx:preprocessor.bzl", "cxx_inherited_preprocessor_infos", - "cxx_merge_cpreprocessors", + "cxx_merge_cpreprocessors_actions", ) load( "@prelude//haskell:library_info.bzl", + "HaskellLibraryProvider", "HaskellLibraryInfoTSet", ) +load( + "@prelude//haskell:library_info.bzl", + "HaskellLibraryInfo", +) +load( + "@prelude//haskell:link_info.bzl", + "HaskellLinkInfo", +) load( "@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", + "HaskellToolchainLibrary", + "DynamicHaskellPackageDbInfo", + "HaskellPackageDbTSet", ) load( "@prelude//haskell:util.bzl", + "attr_deps", "attr_deps_haskell_lib_infos", "attr_deps_haskell_link_infos", + "attr_deps_haskell_toolchain_libraries", "get_artifact_suffix", + "get_source_prefixes", + "is_haskell_boot", "is_haskell_src", "output_extensions", + "src_to_module_name", "srcs_to_pairs", ) load( @@ -32,28 +51,238 @@ load( "LinkStyle", ) load("@prelude//utils:argfile.bzl", "at_argfile") +load("@prelude//:paths.bzl", "paths") +load("@prelude//utils:graph_utils.bzl", "post_order_traversal") +load("@prelude//utils:strings.bzl", "strip_prefix") + +CompiledModuleInfo = provider(fields = { + "abi": provider_field(Artifact), + "interfaces": provider_field(list[Artifact]), + # TODO[AH] track this module's package-name/id & package-db instead. + "db_deps": provider_field(list[Artifact]), +}) + +def _compiled_module_project_as_abi(mod: CompiledModuleInfo) -> cmd_args: + return cmd_args(mod.abi) + +def _compiled_module_project_as_interfaces(mod: CompiledModuleInfo) -> cmd_args: + return cmd_args(mod.interfaces) + +def _compiled_module_reduce_as_packagedb_deps(children: list[dict[Artifact, None]], mod: CompiledModuleInfo | None) -> dict[Artifact, None]: + # TODO[AH] is there a better way to avoid duplicate package-dbs? + # Using a projection instead would produce duplicates. + result = {db: None for db in mod.db_deps} if mod else {} + for child in children: + result.update(child) + return result + +CompiledModuleTSet = transitive_set( + args_projections = { + "abi": _compiled_module_project_as_abi, + "interfaces": _compiled_module_project_as_interfaces, + }, + reductions = { + "packagedb_deps": _compiled_module_reduce_as_packagedb_deps, + }, +) + +DynamicCompileResultInfo = provider(fields = { + "modules": dict[str, CompiledModuleTSet], +}) # The type of the return value of the `_compile()` function. CompileResultInfo = record( - objects = field(Artifact), - hi = field(Artifact), + objects = field(list[Artifact]), + hi = field(list[Artifact]), stubs = field(Artifact), + hashes = field(list[Artifact]), producing_indices = field(bool), -) - -CompileArgsInfo = record( - result = field(CompileResultInfo), - srcs = field(cmd_args), - args_for_cmd = field(cmd_args), - args_for_file = field(cmd_args), + module_tsets = field(DynamicValue), ) PackagesInfo = record( exposed_package_args = cmd_args, packagedb_args = cmd_args, transitive_deps = field(HaskellLibraryInfoTSet), + bin_paths = cmd_args, +) + +_Module = record( + source = field(Artifact), + interfaces = field(list[Artifact]), + hash = field(Artifact), + objects = field(list[Artifact]), + stub_dir = field(Artifact | None), + prefix_dir = field(str), ) + +def _strip_prefix(prefix, s): + stripped = strip_prefix(prefix, s) + + return stripped if stripped != None else s + + +def _modules_by_name(ctx: AnalysisContext, *, sources: list[Artifact], link_style: LinkStyle, enable_profiling: bool, suffix: str) -> dict[str, _Module]: + modules = {} + + osuf, hisuf = output_extensions(link_style, enable_profiling) + + for src in sources: + bootsuf = "" + if is_haskell_boot(src.short_path): + bootsuf = "-boot" + elif not is_haskell_src(src.short_path): + continue + + module_name = src_to_module_name(src.short_path) + bootsuf + interface_path = paths.replace_extension(src.short_path, "." + hisuf + bootsuf) + interface = ctx.actions.declare_output("mod-" + suffix, interface_path) + interfaces = [interface] + object_path = paths.replace_extension(src.short_path, "." + osuf + bootsuf) + object = ctx.actions.declare_output("mod-" + suffix, object_path) + objects = [object] + hash = ctx.actions.declare_output("mod-" + suffix, interface_path + ".hash") + + if link_style in [LinkStyle("static"), LinkStyle("static_pic")]: + dyn_osuf, dyn_hisuf = output_extensions(LinkStyle("shared"), enable_profiling) + interface_path = paths.replace_extension(src.short_path, "." + dyn_hisuf + bootsuf) + interface = ctx.actions.declare_output("mod-" + suffix, interface_path) + interfaces.append(interface) + object_path = paths.replace_extension(src.short_path, "." + dyn_osuf + bootsuf) + object = ctx.actions.declare_output("mod-" + suffix, object_path) + objects.append(object) + + if bootsuf == "": + stub_dir = ctx.actions.declare_output("stub-" + suffix + "-" + module_name, dir=True) + else: + stub_dir = None + + prefix_dir = "mod-" + suffix + + modules[module_name] = _Module( + source = src, + interfaces = interfaces, + hash = hash, + objects = objects, + stub_dir = stub_dir, + prefix_dir = prefix_dir) + + return modules + +def _dynamic_target_metadata_impl(actions, artifacts, dynamic_values, outputs, arg): + # Add -package-db and -package/-expose-package flags for each Haskell + # library dependency. + + packages_info = get_packages_info2( + actions, + arg.deps, + arg.direct_deps_link_info, + arg.haskell_toolchain, + arg.haskell_direct_deps_lib_infos, + LinkStyle("shared"), + specify_pkg_version = False, + enable_profiling = False, + use_empty_lib = True, + for_deps = True, + resolved = dynamic_values, + ) + package_flag = _package_flag(arg.haskell_toolchain) + ghc_args = cmd_args() + ghc_args.add("-hide-all-packages") + ghc_args.add(package_flag, "base") + + ghc_args.add(cmd_args(arg.toolchain_libs, prepend=package_flag)) + ghc_args.add(cmd_args(packages_info.exposed_package_args)) + ghc_args.add(cmd_args(packages_info.packagedb_args, prepend = "-package-db")) + ghc_args.add(arg.compiler_flags) + + md_args = cmd_args(arg.md_gen) + md_args.add(packages_info.bin_paths) + md_args.add("--ghc", arg.haskell_toolchain.compiler) + md_args.add(cmd_args(ghc_args, format="--ghc-arg={}")) + md_args.add( + "--source-prefix", + arg.strip_prefix, + ) + md_args.add(cmd_args(arg.sources, format="--source={}")) + + md_args.add( + arg.lib_package_name_and_prefix, + ) + md_args.add("--output", outputs[arg.md_file].as_output()) + + actions.run(md_args, category = "haskell_metadata", identifier = arg.suffix if arg.suffix else None) + + return [] + +_dynamic_target_metadata = dynamic_actions(impl = _dynamic_target_metadata_impl) + +def target_metadata( + ctx: AnalysisContext, + *, + sources: list[Artifact], + suffix: str = "", + ) -> Artifact: + md_file = ctx.actions.declare_output(ctx.attrs.name + suffix + ".md.json") + md_gen = ctx.attrs._generate_target_metadata[RunInfo] + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + toolchain_libs = [dep.name for dep in attr_deps_haskell_toolchain_libraries(ctx)] + + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( + ctx, + LinkStyle("shared"), + enable_profiling = False, + ) + + # The object and interface file paths are depending on the real module name + # as inferred by GHC, not the source file path; currently this requires the + # module name to correspond to the source file path as otherwise GHC will + # not be able to find the created object or interface files in the search + # path. + # + # (module X.Y.Z must be defined in a file at X/Y/Z.hs) + + ctx.actions.dynamic_output_new(_dynamic_target_metadata( + dynamic = [], + dynamic_values = [haskell_toolchain.packages.dynamic] if haskell_toolchain.packages else [], + outputs = [md_file.as_output()], + arg = struct( + compiler_flags = ctx.attrs.compiler_flags, + deps = ctx.attrs.deps, + direct_deps_link_info = attr_deps_haskell_link_infos(ctx), + haskell_direct_deps_lib_infos = haskell_direct_deps_lib_infos, + haskell_toolchain = haskell_toolchain, + lib_package_name_and_prefix =_attr_deps_haskell_lib_package_name_and_prefix(ctx), + md_file = md_file, + md_gen = md_gen, + sources = sources, + strip_prefix = _strip_prefix(str(ctx.label.cell_root), str(ctx.label.path)), + suffix = suffix, + toolchain_libs = toolchain_libs, + ), + )) + + return md_file + +def _attr_deps_haskell_lib_package_name_and_prefix(ctx: AnalysisContext) -> cmd_args: + args = cmd_args(prepend = "--package") + + for dep in attr_deps(ctx) + ctx.attrs.template_deps: + lib = dep.get(HaskellLibraryProvider) + if lib == None: + continue + + lib_info = lib.lib.values()[0] + args.add(cmd_args( + lib_info.name, + cmd_args(lib_info.db, parent = 1), + delimiter = ":", + )) + + return args + def _package_flag(toolchain: HaskellToolchainInfo) -> str: if toolchain.support_expose_package: return "-expose-package" @@ -64,13 +293,45 @@ def get_packages_info( ctx: AnalysisContext, link_style: LinkStyle, specify_pkg_version: bool, - enable_profiling: bool) -> PackagesInfo: + enable_profiling: bool, + use_empty_lib: bool) -> PackagesInfo: haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( + ctx, + link_style, + enable_profiling, + ) + + return get_packages_info2( + actions = ctx.actions, + deps = [], + direct_deps_link_info = attr_deps_haskell_link_infos(ctx), + haskell_toolchain = haskell_toolchain, + haskell_direct_deps_lib_infos = haskell_direct_deps_lib_infos, + link_style = link_style, + specify_pkg_version = specify_pkg_version, + enable_profiling = enable_profiling, + use_empty_lib = use_empty_lib, + resolved = {}, + ) + +def get_packages_info2( + actions: AnalysisActions, + deps: list[Dependency], + direct_deps_link_info: list[HaskellLinkInfo], + haskell_toolchain: HaskellToolchainInfo, + haskell_direct_deps_lib_infos: list[HaskellLibraryInfo], + link_style: LinkStyle, + specify_pkg_version: bool, + enable_profiling: bool, + use_empty_lib: bool, + resolved: dict[DynamicValue, ResolvedDynamicValue], + for_deps: bool = False) -> PackagesInfo: + # Collect library dependencies. Note that these don't need to be in a # particular order. - direct_deps_link_info = attr_deps_haskell_link_infos(ctx) - libs = ctx.actions.tset( + libs = actions.tset( HaskellLibraryInfoTSet, children = [ lib.prof_info[link_style] if enable_profiling else lib.info[link_style] @@ -80,13 +341,21 @@ def get_packages_info( # base is special and gets exposed by default package_flag = _package_flag(haskell_toolchain) + exposed_package_args = cmd_args([package_flag, "base"]) + if for_deps: + get_db = lambda l: l.deps_db + elif use_empty_lib: + get_db = lambda l: l.empty_db + else: + get_db = lambda l: l.db + packagedb_args = cmd_args() packagedb_set = {} for lib in libs.traverse(): - packagedb_set[lib.db] = None + packagedb_set[get_db(lib)] = None hidden_args = cmd_args(hidden = [ lib.import_dirs.values(), lib.stub_dirs, @@ -94,20 +363,36 @@ def get_packages_info( # we're using Template Haskell: lib.libs, ]) - exposed_package_args.add(hidden_args) - packagedb_args.add(hidden_args) + if resolved: + pkg_deps = resolved[haskell_toolchain.packages.dynamic] + package_db = pkg_deps.providers[DynamicHaskellPackageDbInfo].packages + else: + package_db = {} + + direct_toolchain_libs = [ + dep[HaskellToolchainLibrary].name + for dep in deps + if HaskellToolchainLibrary in dep + ] + + toolchain_libs = direct_toolchain_libs + libs.reduce("packages") + + package_db_tset = actions.tset( + HaskellPackageDbTSet, + children = [package_db[name] for name in toolchain_libs if name in package_db] + ) + # These we need to add for all the packages/dependencies, i.e. # direct and transitive (e.g. `fbcode-common-hs-util-hs-array`) - packagedb_args.add([cmd_args("-package-db", x) for x in packagedb_set]) + packagedb_args.add(packagedb_set.keys()) - haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( - ctx, - link_style, - enable_profiling, - ) + packagedb_args.add(package_db_tset.project_as_args("package_db")) + + direct_package_paths = [package_db[name].value.path for name in direct_toolchain_libs if name in package_db] + bin_paths = cmd_args(direct_package_paths, format="--bin-path={}/bin") # Expose only the packages we depend on directly for lib in haskell_direct_deps_lib_infos: @@ -121,146 +406,484 @@ def get_packages_info( exposed_package_args = exposed_package_args, packagedb_args = packagedb_args, transitive_deps = libs, + bin_paths = bin_paths, ) -def compile_args( - ctx: AnalysisContext, - link_style: LinkStyle, - enable_profiling: bool, - pkgname = None, - suffix: str = "") -> CompileArgsInfo: - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] +CommonCompileModuleArgs = record( + command = field(cmd_args), + args_for_file = field(cmd_args), + package_env_args = field(cmd_args), +) - compile_cmd = cmd_args() - compile_cmd.add(haskell_toolchain.compiler_flags) +def _common_compile_module_args( + actions: AnalysisActions, + *, + compiler_flags: list[ArgLike], + ghc_wrapper: RunInfo, + haskell_toolchain: HaskellToolchainInfo, + resolved: dict[DynamicValue, ResolvedDynamicValue], + enable_haddock: bool, + enable_profiling: bool, + link_style: LinkStyle, + main: None | str, + label: Label, + deps: list[Dependency], + external_tool_paths: list[RunInfo], + sources: list[Artifact], + direct_deps_info: list[HaskellLibraryInfoTSet], + pkgname: str | None = None, +) -> CommonCompileModuleArgs: + command = cmd_args(ghc_wrapper) + command.add("--ghc", haskell_toolchain.compiler) # Some rules pass in RTS (e.g. `+RTS ... -RTS`) options for GHC, which can't # be parsed when inside an argsfile. - compile_cmd.add(ctx.attrs.compiler_flags) + command.add(haskell_toolchain.compiler_flags) + command.add(compiler_flags) + + command.add("-c") + + if main != None: + command.add(["-main-is", main]) + + if enable_haddock: + command.add("-haddock") + + non_haskell_sources = [ + src + for (path, src) in srcs_to_pairs(sources) + if not is_haskell_src(path) and not is_haskell_boot(path) + ] - compile_args = cmd_args() - compile_args.add("-no-link", "-i") + if non_haskell_sources: + warning("{} specifies non-haskell file in `srcs`, consider using `srcs_deps` instead".format(label)) + + args_for_file = cmd_args(hidden = non_haskell_sources) + + args_for_file.add("-no-link", "-i") + args_for_file.add("-hide-all-packages") if enable_profiling: - compile_args.add("-prof") + args_for_file.add("-prof") if link_style == LinkStyle("shared"): - compile_args.add("-dynamic", "-fPIC") + args_for_file.add("-dynamic", "-fPIC") elif link_style == LinkStyle("static_pic"): - compile_args.add("-fPIC", "-fexternal-dynamic-refs") + args_for_file.add("-fPIC", "-fexternal-dynamic-refs") osuf, hisuf = output_extensions(link_style, enable_profiling) - compile_args.add("-osuf", osuf, "-hisuf", hisuf) - - if getattr(ctx.attrs, "main", None) != None: - compile_args.add(["-main-is", ctx.attrs.main]) + args_for_file.add("-osuf", osuf, "-hisuf", hisuf) - artifact_suffix = get_artifact_suffix(link_style, enable_profiling, suffix) + # Add args from preprocess-able inputs. + inherited_pre = cxx_inherited_preprocessor_infos(deps) + pre = cxx_merge_cpreprocessors_actions(actions, [], inherited_pre) + pre_args = pre.set.project_as_args("args") + args_for_file.add(cmd_args(pre_args, format = "-optP={}")) - objects = ctx.actions.declare_output( - "objects-" + artifact_suffix, - dir = True, - ) - hi = ctx.actions.declare_output("hi-" + artifact_suffix, dir = True) - stubs = ctx.actions.declare_output("stubs-" + artifact_suffix, dir = True) - - compile_args.add( - "-odir", - objects.as_output(), - "-hidir", - hi.as_output(), - "-hiedir", - hi.as_output(), - "-stubdir", - stubs.as_output(), - ) + if pkgname: + args_for_file.add(["-this-unit-id", pkgname]) # Add -package-db and -package/-expose-package flags for each Haskell # library dependency. - packages_info = get_packages_info( - ctx, - link_style, - specify_pkg_version = False, - enable_profiling = enable_profiling, - ) - compile_args.add(packages_info.exposed_package_args) - compile_args.add(packages_info.packagedb_args) + libs = actions.tset(HaskellLibraryInfoTSet, children = direct_deps_info) - # Add args from preprocess-able inputs. - inherited_pre = cxx_inherited_preprocessor_infos(ctx.attrs.deps) - pre = cxx_merge_cpreprocessors(ctx, [], inherited_pre) - pre_args = pre.set.project_as_args("args") - compile_args.add(cmd_args(pre_args, format = "-optP={}")) + direct_toolchain_libs = [ + dep[HaskellToolchainLibrary].name + for dep in deps + if HaskellToolchainLibrary in dep + ] + toolchain_libs = direct_toolchain_libs + libs.reduce("packages") - if pkgname: - compile_args.add(["-this-unit-id", pkgname]) + if haskell_toolchain.packages: + pkg_deps = resolved[haskell_toolchain.packages.dynamic] + package_db = pkg_deps.providers[DynamicHaskellPackageDbInfo].packages + else: + package_db = [] + + package_db_tset = actions.tset( + HaskellPackageDbTSet, + children = [package_db[name] for name in toolchain_libs if name in package_db] + ) - arg_srcs = [] - hidden_srcs = [] + direct_package_paths = [package_db[name].value.path for name in direct_toolchain_libs if name in package_db] + args_for_file.add(cmd_args( + direct_package_paths, + format="--bin-path={}/bin", + )) + + args_for_file.add(cmd_args( + external_tool_paths, + format="--bin-exe={}", + )) + + packagedb_args = cmd_args(libs.project_as_args("empty_package_db")) + packagedb_args.add(package_db_tset.project_as_args("package_db")) + + # TODO[AH] Avoid duplicates and share identical env files. + # The set of package-dbs can be known at the package level, not just the + # module level. So, we could generate this file outside of the + # dynamic_output action. + package_env_file = actions.declare_output(".".join([ + label.name, + "package-db", + output_extensions(link_style, enable_profiling)[1], + "env", + ])) + package_env = cmd_args(delimiter = "\n") + package_env.add(cmd_args( + packagedb_args, + format = "package-db {}", + ).relative_to(package_env_file, parent = 1)) + actions.write( + package_env_file, + package_env, + ) + package_env_args = cmd_args( + package_env_file, + prepend = "-package-env", + hidden = packagedb_args, + ) - aux_deps = ctx.attrs.srcs_deps.get(module.source) - if aux_deps: - hidden_srcs(aux_deps) + return CommonCompileModuleArgs( + command = command, + args_for_file = args_for_file, + package_env_args = package_env_args, + ) - for (path, src) in srcs_to_pairs(ctx.attrs.srcs): - # hs-boot files aren't expected to be an argument to compiler but does need - # to be included in the directory of the associated src file - if is_haskell_src(path): - arg_srcs.append(src) +def _compile_module( + actions: AnalysisActions, + *, + common_args: CommonCompileModuleArgs, + link_style: LinkStyle, + enable_profiling: bool, + enable_th: bool, + haskell_toolchain: HaskellToolchainInfo, + label: Label, + module_name: str, + module: _Module, + module_tsets: dict[str, CompiledModuleTSet], + md_file: Artifact, + graph: dict[str, list[str]], + package_deps: dict[str, list[str]], + outputs: dict[Artifact, Artifact], + artifact_suffix: str, + direct_deps_by_name: dict[str, typing.Any], + toolchain_deps_by_name: dict[str, None], + aux_deps: None | list[Artifact], + src_envs: None | dict[str, ArgLike], + source_prefixes: list[str], +) -> CompiledModuleTSet: + # These compiler arguments can be passed in a response file. + compile_args_for_file = cmd_args(common_args.args_for_file, hidden = aux_deps or []) + + packagedb_tag = actions.artifact_tag() + compile_args_for_file.add(packagedb_tag.tag_artifacts(common_args.package_env_args)) + + dep_file = actions.declare_output(".".join([ + label.name, + module_name or "pkg", + "package-db", + output_extensions(link_style, enable_profiling)[1], + "dep", + ])).as_output() + tagged_dep_file = packagedb_tag.tag_artifacts(dep_file) + compile_args_for_file.add("--buck2-packagedb-dep", tagged_dep_file) + + objects = [outputs[obj] for obj in module.objects] + his = [outputs[hi] for hi in module.interfaces] + + compile_args_for_file.add("-o", objects[0].as_output()) + compile_args_for_file.add("-ohi", his[0].as_output()) + + # Set the output directories. We do not use the -outputdir flag, but set the directories individually. + # Note, the -outputdir option is shorthand for the combination of -odir, -hidir, -hiedir, -stubdir and -dumpdir. + # But setting -hidir effectively disables the use of the search path to look up interface files, + # as ghc exclusively looks in that directory when it is set. + for dir in ["o", "hie", "dump"]: + compile_args_for_file.add( + "-{}dir".format(dir), cmd_args([cmd_args(md_file, ignore_artifacts=True, parent=1), module.prefix_dir], delimiter="/"), + ) + if module.stub_dir != None: + stubs = outputs[module.stub_dir] + compile_args_for_file.add("-stubdir", stubs.as_output()) + + if link_style in [LinkStyle("static_pic"), LinkStyle("static")]: + compile_args_for_file.add("-dynamic-too") + compile_args_for_file.add("-dyno", objects[1].as_output()) + compile_args_for_file.add("-dynohi", his[1].as_output()) + + compile_args_for_file.add(module.source) + + abi_tag = actions.artifact_tag() + + toolchain_deps = [] + library_deps = [] + exposed_package_modules = [] + exposed_package_dbs = [] + for dep_pkgname, dep_modules in package_deps.items(): + if dep_pkgname in toolchain_deps_by_name: + toolchain_deps.append(dep_pkgname) + elif dep_pkgname in direct_deps_by_name: + library_deps.append(dep_pkgname) + exposed_package_dbs.append(direct_deps_by_name[dep_pkgname].package_db) + for dep_modname in dep_modules: + exposed_package_modules.append(direct_deps_by_name[dep_pkgname].modules[dep_modname]) else: - hidden_srcs.append(src) - srcs = cmd_args( - arg_srcs, - hidden = hidden_srcs, + fail("Unknown library dependency '{}'. Add the library to the `deps` attribute".format(dep_pkgname)) + + # Transitive module dependencies from other packages. + cross_package_modules = actions.tset( + CompiledModuleTSet, + children = exposed_package_modules, ) + # Transitive module dependencies from the same package. + this_package_modules = [ + module_tsets[dep_name] + for dep_name in graph[module_name] + ] + + dependency_modules = actions.tset( + CompiledModuleTSet, + children = [cross_package_modules] + this_package_modules, + ) + + compile_cmd_args = [common_args.command] + compile_cmd_hidden = [ + abi_tag.tag_artifacts(dependency_modules.project_as_args("interfaces")), + dependency_modules.project_as_args("abi"), + ] + if src_envs: + for k, v in src_envs.items(): + compile_args_for_file.add(cmd_args( + k, + format="--extra-env-key={}", + )) + compile_args_for_file.add(cmd_args( + v, + format="--extra-env-value={}", + )) + if haskell_toolchain.use_argsfile: + compile_cmd_args.append(at_argfile( + actions = actions, + name = "haskell_compile_" + artifact_suffix + ".argsfile", + args = compile_args_for_file, + allow_args = True, + )) + else: + compile_cmd_args.append(compile_args_for_file) - producing_indices = "-fwrite-ide-info" in ctx.attrs.compiler_flags + compile_cmd = cmd_args(compile_cmd_args, hidden = compile_cmd_hidden) - return CompileArgsInfo( - result = CompileResultInfo( - objects = objects, - hi = hi, - stubs = stubs, - producing_indices = producing_indices, + # add each module dir prefix to search path + for prefix in source_prefixes: + compile_cmd.add( + cmd_args( + cmd_args(md_file, format = "-i{}", ignore_artifacts=True, parent=1), + "/", + paths.join(module.prefix_dir, prefix), + delimiter="" + ) + ) + + + compile_cmd.add(cmd_args(library_deps, prepend = "-package")) + compile_cmd.add(cmd_args(toolchain_deps, prepend = "-package")) + + compile_cmd.add("-fbyte-code-and-object-code") + if enable_th: + compile_cmd.add("-fprefer-byte-code") + + compile_cmd.add(cmd_args(dependency_modules.reduce("packagedb_deps").keys(), prepend = "--buck2-package-db")) + + dep_file = actions.declare_output("dep-{}_{}".format(module_name, artifact_suffix)).as_output() + + tagged_dep_file = abi_tag.tag_artifacts(dep_file) + + compile_cmd.add("--buck2-dep", tagged_dep_file) + compile_cmd.add("--abi-out", outputs[module.hash].as_output()) + + actions.run( + compile_cmd, category = "haskell_compile_" + artifact_suffix.replace("-", "_"), identifier = module_name, + dep_files = { + "abi": abi_tag, + "packagedb": packagedb_tag, + } + ) + + module_tset = actions.tset( + CompiledModuleTSet, + value = CompiledModuleInfo( + abi = module.hash, + interfaces = module.interfaces, + db_deps = exposed_package_dbs, ), - srcs = srcs, - args_for_cmd = compile_cmd, - args_for_file = compile_args, + children = [cross_package_modules] + this_package_modules, + ) + + return module_tset + +def _dynamic_do_compile_impl(actions, artifacts, dynamic_values, outputs, arg): + direct_deps_by_name = { + info.value.name: struct( + package_db = info.value.empty_db, + modules = dynamic_values[info.value.dynamic[arg.enable_profiling]].providers[DynamicCompileResultInfo].modules, + ) + for info in arg.direct_deps_info + } + common_args = _common_compile_module_args( + actions, + compiler_flags = arg.compiler_flags, + deps = arg.deps, + external_tool_paths = arg.external_tool_paths, + ghc_wrapper = arg.ghc_wrapper, + haskell_toolchain = arg.haskell_toolchain, + label = arg.label, + main = arg.main, + resolved = dynamic_values, + sources = arg.sources, + enable_haddock = arg.enable_haddock, + enable_profiling = arg.enable_profiling, + link_style = arg.link_style, + direct_deps_info = arg.direct_deps_info, + pkgname = arg.pkgname, ) + md = artifacts[arg.md_file].read_json() + th_modules = md["th_modules"] + module_map = md["module_mapping"] + graph = md["module_graph"] + package_deps = md["package_deps"] + + mapped_modules = { module_map.get(k, k): v for k, v in arg.modules.items() } + module_tsets = {} + source_prefixes = get_source_prefixes(arg.sources, module_map) + + for module_name in post_order_traversal(graph): + module = mapped_modules[module_name] + module_tsets[module_name] = _compile_module( + actions, + aux_deps = arg.sources_deps.get(module.source), + src_envs = arg.srcs_envs.get(module.source), + common_args = common_args, + link_style = arg.link_style, + enable_profiling = arg.enable_profiling, + enable_th = module_name in th_modules, + haskell_toolchain = arg.haskell_toolchain, + label = arg.label, + module_name = module_name, + module = module, + module_tsets = module_tsets, + graph = graph, + package_deps = package_deps.get(module_name, {}), + outputs = outputs, + md_file = arg.md_file, + artifact_suffix = arg.artifact_suffix, + direct_deps_by_name = direct_deps_by_name, + toolchain_deps_by_name = arg.toolchain_deps_by_name, + source_prefixes = source_prefixes, + ) + + return [DynamicCompileResultInfo(modules = module_tsets)] + + + +_dynamic_do_compile = dynamic_actions(impl = _dynamic_do_compile_impl) + # Compile all the context's sources. def compile( ctx: AnalysisContext, link_style: LinkStyle, enable_profiling: bool, + enable_haddock: bool, + md_file: Artifact, pkgname: str | None = None) -> CompileResultInfo: - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - compile_cmd = cmd_args(haskell_toolchain.compiler) + artifact_suffix = get_artifact_suffix(link_style, enable_profiling) + + modules = _modules_by_name(ctx, sources = ctx.attrs.srcs, link_style = link_style, enable_profiling = enable_profiling, suffix = artifact_suffix) - args = compile_args(ctx, link_style, enable_profiling, pkgname) + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - compile_cmd.add(args.args_for_cmd) + interfaces = [interface for module in modules.values() for interface in module.interfaces] + objects = [object for module in modules.values() for object in module.objects] + stub_dirs = [ + module.stub_dir + for module in modules.values() + if module.stub_dir != None + ] + abi_hashes = [module.hash for module in modules.values()] - artifact_suffix = get_artifact_suffix(link_style, enable_profiling) + # Collect library dependencies. Note that these don't need to be in a + # particular order. + toolchain_deps_by_name = { + lib.name: None + for lib in attr_deps_haskell_toolchain_libraries(ctx) + } + direct_deps_info = [ + lib.prof_info[link_style] if enable_profiling else lib.info[link_style] + for lib in attr_deps_haskell_link_infos(ctx) + ] + + dyn_module_tsets = ctx.actions.dynamic_output_new(_dynamic_do_compile( + dynamic = [md_file], + dynamic_values = [ + info.value.dynamic[enable_profiling] + for lib in attr_deps_haskell_link_infos(ctx) + for info in [ + lib.prof_info[link_style] + if enable_profiling else + lib.info[link_style] + ] + ] + ([ haskell_toolchain.packages.dynamic ] if haskell_toolchain.packages else [ ]), + outputs = [o.as_output() for o in interfaces + objects + stub_dirs + abi_hashes], + arg = struct( + artifact_suffix = artifact_suffix, + compiler_flags = ctx.attrs.compiler_flags, + deps = ctx.attrs.deps, + direct_deps_info = direct_deps_info, + enable_haddock = enable_haddock, + enable_profiling = enable_profiling, + external_tool_paths = [tool[RunInfo] for tool in ctx.attrs.external_tools], + ghc_wrapper = ctx.attrs._ghc_wrapper[RunInfo], + haskell_toolchain = haskell_toolchain, + label = ctx.label, + link_style = link_style, + main = getattr(ctx.attrs, "main", None), + md_file = md_file, + modules = modules, + pkgname = pkgname, + sources = ctx.attrs.srcs, + sources_deps = ctx.attrs.srcs_deps, + srcs_envs = ctx.attrs.srcs_envs, + toolchain_deps_by_name = toolchain_deps_by_name, + ), + )) - if args.args_for_file: - if haskell_toolchain.use_argsfile: - compile_cmd.add(at_argfile( - actions = ctx.actions, - name = artifact_suffix + ".haskell_compile_argsfile", - args = [args.args_for_file, args.srcs], - allow_args = True, - )) - else: - compile_cmd.add(args.args_for_file) - compile_cmd.add(args.srcs) + stubs_dir = ctx.actions.declare_output("stubs-" + artifact_suffix, dir=True) - artifact_suffix = get_artifact_suffix(link_style, enable_profiling) + # collect the stubs from all modules into the stubs_dir ctx.actions.run( - compile_cmd, - category = "haskell_compile_" + artifact_suffix.replace("-", "_"), - no_outputs_cleanup = True, + cmd_args([ + "bash", "-exuc", + """\ + mkdir -p \"$0\" + for stub; do + find \"$stub\" -mindepth 1 -maxdepth 1 -exec cp -r -t \"$0\" '{}' ';' + done + """, + stubs_dir.as_output(), + stub_dirs + ]), + category = "haskell_stubs", + identifier = artifact_suffix, + local_only = True, ) - return args.result + return CompileResultInfo( + objects = objects, + hi = interfaces, + hashes = abi_hashes, + stubs = stubs_dir, + producing_indices = False, + module_tsets = dyn_module_tsets, + ) diff --git a/haskell/haskell.bzl b/haskell/haskell.bzl index 975ea03ad..94bf73df8 100644 --- a/haskell/haskell.bzl +++ b/haskell/haskell.bzl @@ -7,6 +7,7 @@ # Implementation of the Haskell build rules. +load("@prelude//utils:arglike.bzl", "ArgLike") load("@prelude//:paths.bzl", "paths") load("@prelude//cxx:archive.bzl", "make_archive") load( @@ -54,6 +55,8 @@ load( "@prelude//haskell:compile.bzl", "CompileResultInfo", "compile", + "get_packages_info2", + "target_metadata", ) load( "@prelude//haskell:haskell_haddock.bzl", @@ -75,19 +78,24 @@ load( load( "@prelude//haskell:toolchain.bzl", "HaskellToolchainInfo", + "HaskellToolchainLibrary", + "HaskellPackageDbTSet", + "DynamicHaskellPackageDbInfo", ) load( "@prelude//haskell:util.bzl", "attr_deps", "attr_deps_haskell_link_infos_sans_template_deps", + "attr_deps_haskell_lib_infos", + "attr_deps_haskell_link_infos", + "attr_deps_haskell_toolchain_libraries", "attr_deps_merged_link_infos", "attr_deps_profiling_link_infos", "attr_deps_shared_library_infos", "get_artifact_suffix", - "is_haskell_src", "output_extensions", "src_to_module_name", - "srcs_to_pairs", + "get_source_prefixes", ) load( "@prelude//linking:link_groups.bzl", @@ -238,6 +246,7 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: import_dirs = {}, stub_dirs = [], id = ctx.attrs.id, + dynamic = None, libs = libs, version = ctx.attrs.version, is_prebuilt = True, @@ -249,6 +258,7 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: import_dirs = {}, stub_dirs = [], id = ctx.attrs.id, + dynamic = None, libs = prof_libs, version = ctx.attrs.version, is_prebuilt = True, @@ -377,24 +387,18 @@ def haskell_prebuilt_library_impl(ctx: AnalysisContext) -> list[Provider]: linkable_graph, ] -def _srcs_to_objfiles( - ctx: AnalysisContext, - odir: Artifact, - osuf: str) -> list[Artifact]: - objfiles = [] - for src, _ in srcs_to_pairs(ctx.attrs.srcs): - # Don't link boot sources, as they're only meant to be used for compiling. - if is_haskell_src(src): - objfiles.append(odir.project(paths.replace_extension(src, "." + osuf))) - return objfiles - +# Script to generate a GHC package-db entry for a new package. +# +# Sets --force so that ghc-pkg does not check for .hi, .so, ... files. +# This way package actions can be scheduled before actual build actions, +# don't lie on the critical path for a build, and don't form a bottleneck. _REGISTER_PACKAGE = """\ set -eu GHC_PKG=$1 DB=$2 PKGCONF=$3 "$GHC_PKG" init "$DB" -"$GHC_PKG" register --package-conf "$DB" --no-expand-pkgroot "$PKGCONF" +"$GHC_PKG" register --package-conf "$DB" --no-expand-pkgroot "$PKGCONF" --force """ # Create a package @@ -412,85 +416,114 @@ PKGCONF=$3 # - controlling module visibility: only dependencies that are # directly declared as dependencies may be used # -# - Template Haskell: the compiler needs to load libraries itself -# at compile time, so it uses the package specs to find out -# which libraries and where. +# - by GHCi when loading packages into the repl +# +# - when linking binaries statically, in order to pass libraries +# to the linker in the correct order def _make_package( ctx: AnalysisContext, link_style: LinkStyle, pkgname: str, - libname: str, + libname: str | None, hlis: list[HaskellLibraryInfo], - hi: dict[bool, Artifact], - lib: dict[bool, Artifact], - enable_profiling: bool) -> Artifact: + profiling: list[bool], + enable_profiling: bool, + use_empty_lib: bool, + md_file: Artifact, + for_deps: bool = False) -> Artifact: artifact_suffix = get_artifact_suffix(link_style, enable_profiling) - # Don't expose boot sources, as they're only meant to be used for compiling. - modules = [src_to_module_name(x) for x, _ in srcs_to_pairs(ctx.attrs.srcs) if is_haskell_src(x)] + def mk_artifact_dir(dir_prefix: str, profiled: bool, subdir: str = "") -> str: + suffix = get_artifact_suffix(link_style, profiled) + if subdir: + suffix = paths.join(suffix, subdir) + return "\"${pkgroot}/" + dir_prefix + "-" + suffix + "\"" + + if for_deps: + pkg_conf = ctx.actions.declare_output("pkg-" + artifact_suffix + "_deps.conf") + db = ctx.actions.declare_output("db-" + artifact_suffix + "_deps", dir = True) + elif use_empty_lib: + pkg_conf = ctx.actions.declare_output("pkg-" + artifact_suffix + "_empty.conf") + db = ctx.actions.declare_output("db-" + artifact_suffix + "_empty", dir = True) + else: + pkg_conf = ctx.actions.declare_output("pkg-" + artifact_suffix + ".conf") + db = ctx.actions.declare_output("db-" + artifact_suffix, dir = True) - if enable_profiling: - # Add the `-p` suffix otherwise ghc will look for objects - # following this logic (https://fburl.com/code/3gmobm5x) and will fail. - libname += "_p" + def write_package_conf(ctx, artifacts, outputs, md_file=md_file, libname=libname): + md = artifacts[md_file].read_json() + module_map = md["module_mapping"] - def mk_artifact_dir(dir_prefix: str, profiled: bool) -> str: - art_suff = get_artifact_suffix(link_style, profiled) - return "\"${pkgroot}/" + dir_prefix + "-" + art_suff + "\"" + source_prefixes = get_source_prefixes(ctx.attrs.srcs, module_map) - import_dirs = [ - mk_artifact_dir("hi", profiled) - for profiled in hi.keys() - ] - library_dirs = [ - mk_artifact_dir("lib", profiled) - for profiled in hi.keys() - ] + modules = [ + module + for module in md["module_graph"].keys() + if not module.endswith("-boot") + ] - conf = [ - "name: " + pkgname, - "version: 1.0.0", - "id: " + pkgname, - "key: " + pkgname, - "exposed: False", - "exposed-modules: " + ", ".join(modules), - "import-dirs:" + ", ".join(import_dirs), - "library-dirs:" + ", ".join(library_dirs), - "extra-libraries: " + libname, - "depends: " + ", ".join([lib.id for lib in hlis]), - ] - pkg_conf = ctx.actions.write("pkg-" + artifact_suffix + ".conf", conf) + # XXX use a single import dir when this package db is used for resolving dependencies with ghc -M, + # which works around an issue with multiple import dirs resulting in GHC trying to locate interface files + # for each exposed module + import_dirs = ["."] if for_deps else [ + mk_artifact_dir("mod", profiled, src_prefix) for profiled in profiling for src_prefix in source_prefixes + ] - db = ctx.actions.declare_output("db-" + artifact_suffix) + conf = [ + "name: " + pkgname, + "version: 1.0.0", + "id: " + pkgname, + "key: " + pkgname, + "exposed: False", + "exposed-modules: " + ", ".join(modules), + "import-dirs:" + ", ".join(import_dirs), + "depends: " + ", ".join([lib.id for lib in hlis]), + ] - # While the list of hlis is unique, there may be multiple packages in the same db. - # Cutting down the GHC_PACKAGE_PATH significantly speeds up GHC. - db_deps = {x.db: None for x in hlis}.keys() + if not use_empty_lib: + if not libname: + fail("argument `libname` cannot be empty, when use_empty_lib == False") - # So that ghc-pkg can find the DBs for the dependencies. We might - # be able to use flags for this instead, but this works. - ghc_package_path = cmd_args( - db_deps, - delimiter = ":", - ) + if enable_profiling: + # Add the `-p` suffix otherwise ghc will look for objects + # following this logic (https://fburl.com/code/3gmobm5x) and will fail. + libname += "_p" - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - ctx.actions.run( - cmd_args( - [ + library_dirs = [mk_artifact_dir("lib", profiled) for profiled in profiling] + conf.append("library-dirs:" + ", ".join(library_dirs)) + conf.append("extra-libraries: " + libname) + + ctx.actions.write(outputs[pkg_conf].as_output(), conf) + + db_deps = [x.db for x in hlis] + + # So that ghc-pkg can find the DBs for the dependencies. We might + # be able to use flags for this instead, but this works. + ghc_package_path = cmd_args( + db_deps, + delimiter = ":", + ) + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + ctx.actions.run( + cmd_args([ "sh", "-c", _REGISTER_PACKAGE, "", haskell_toolchain.packager, - db.as_output(), + outputs[db].as_output(), pkg_conf, - ], - # needs hi, because ghc-pkg checks that the .hi files exist - hidden = hi.values() + lib.values(), - ), - category = "haskell_package_" + artifact_suffix.replace("-", "_"), - env = {"GHC_PACKAGE_PATH": ghc_package_path} if db_deps else {}, + ]), + category = "haskell_package_" + artifact_suffix.replace("-", "_"), + identifier = "empty" if use_empty_lib else "final", + env = {"GHC_PACKAGE_PATH": ghc_package_path} if db_deps else {}, + ) + + ctx.actions.dynamic_output( + dynamic = [md_file], + inputs = [], + outputs = [pkg_conf.as_output(), db.as_output()], + f = write_package_conf ) return db @@ -516,6 +549,59 @@ def _get_haskell_shared_library_name_linker_flags( else: fail("Unknown linker type '{}'.".format(linker_type)) +def _dynamic_link_shared_impl(actions, artifacts, dynamic_values, outputs, arg): + pkg_deps = dynamic_values[arg.haskell_toolchain.packages.dynamic] + package_db = pkg_deps.providers[DynamicHaskellPackageDbInfo].packages + + package_db_tset = actions.tset( + HaskellPackageDbTSet, + children = [package_db[name] for name in arg.toolchain_libs if name in package_db] + ) + + link_args = cmd_args() + link_cmd_args = [cmd_args(arg.haskell_toolchain.linker)] + link_cmd_hidden = [] + + link_args.add(arg.haskell_toolchain.linker_flags) + link_args.add(arg.linker_flags) + link_args.add("-hide-all-packages") + link_args.add(cmd_args(arg.toolchain_libs, prepend = "-package")) + link_args.add(cmd_args(package_db_tset.project_as_args("package_db"), prepend="-package-db")) + link_args.add( + get_shared_library_flags(arg.linker_info.type), + "-dynamic", + cmd_args( + _get_haskell_shared_library_name_linker_flags(arg.linker_info.type, arg.libfile), + prepend = "-optl", + ), + ) + + link_args.add(arg.objects) + + link_args.add(cmd_args(unpack_link_args(arg.infos), prepend = "-optl")) + + if arg.use_argsfile_at_link: + link_cmd_args.append(at_argfile( + actions = actions, + name = "haskell_link_" + arg.artifact_suffix.replace("-", "_") + ".argsfile", + args = link_args, + allow_args = True, + )) + else: + link_cmd_args.append(link_args) + + link_cmd = cmd_args(link_cmd_args, hidden = link_cmd_hidden) + link_cmd.add("-o", outputs[arg.lib].as_output()) + + actions.run( + link_cmd, + category = "haskell_link" + arg.artifact_suffix.replace("-", "_"), + ) + + return [] + +_dynamic_link_shared = dynamic_actions(impl = _dynamic_link_shared_impl) + def _build_haskell_lib( ctx, libname: str, @@ -524,6 +610,8 @@ def _build_haskell_lib( nlis: list[MergedLinkInfo], # native link infos from all deps link_style: LinkStyle, enable_profiling: bool, + enable_haddock: bool, + md_file: Artifact, # The non-profiling artifacts are also needed to build the package for # profiling, so it should be passed when `enable_profiling` is True. non_profiling_hlib: [HaskellLibBuildOutput, None] = None) -> HaskellLibBuildOutput: @@ -532,13 +620,13 @@ def _build_haskell_lib( # Link the objects into a library haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - osuf, _hisuf = output_extensions(link_style, enable_profiling) - # Compile the sources compiled = compile( ctx, link_style, enable_profiling = enable_profiling, + enable_haddock = enable_haddock, + md_file = md_file, pkgname = pkgname, ) solibs = {} @@ -559,37 +647,39 @@ def _build_haskell_lib( # only gather direct dependencies uniq_infos = [x[link_style].value for x in linfos] - objfiles = _srcs_to_objfiles(ctx, compiled.objects, osuf) + toolchain_libs = [dep.name for dep in attr_deps_haskell_toolchain_libraries(ctx)] if link_style == LinkStyle("shared"): lib = ctx.actions.declare_output(lib_short_path) - link = cmd_args( - [haskell_toolchain.linker] + - [haskell_toolchain.linker_flags] + - [ctx.attrs.linker_flags] + - ["-o", lib.as_output()] + - [ - get_shared_library_flags(linker_info.type), - "-dynamic", - cmd_args( - _get_haskell_shared_library_name_linker_flags(linker_info.type, libfile), - prepend = "-optl", - ), - ] + - [objfiles], - hidden = compiled.stubs, - ) + objects = [ + object + for object in compiled.objects + if not object.extension.endswith("-boot") + ] infos = get_link_args_for_strategy( ctx, nlis, to_link_strategy(link_style), ) - link.add(cmd_args(unpack_link_args(infos), prepend = "-optl")) - ctx.actions.run( - link, - category = "haskell_link" + artifact_suffix.replace("-", "_"), - ) + + ctx.actions.dynamic_output_new(_dynamic_link_shared( + dynamic = [], + dynamic_values = [haskell_toolchain.packages.dynamic], + outputs = [lib.as_output()], + arg = struct( + artifact_suffix = artifact_suffix, + haskell_toolchain = haskell_toolchain, + infos = infos, + lib = lib, + libfile = libfile, + linker_flags = ctx.attrs.linker_flags, + linker_info = linker_info, + objects = objects, + toolchain_libs = toolchain_libs, + use_argsfile_at_link = ctx.attrs.use_argsfile_at_link, + ), + )) solibs[libfile] = LinkedObject(output = lib, unstripped_output = lib) libs = [lib] @@ -600,7 +690,7 @@ def _build_haskell_lib( else: # static flavours # TODO: avoid making an archive for a single object, like cxx does # (but would that work with Template Haskell?) - archive = make_archive(ctx, lib_short_path, objfiles) + archive = make_archive(ctx, lib_short_path, compiled.objects) lib = archive.artifact libs = [lib] + archive.external_objects link_infos = LinkInfos( @@ -619,22 +709,29 @@ def _build_haskell_lib( if not non_profiling_hlib: fail("Non-profiling HaskellLibBuildOutput wasn't provided when building profiling lib") + dynamic = { + True: compiled.module_tsets, + False: non_profiling_hlib.compiled.module_tsets, + } import_artifacts = { True: compiled.hi, False: non_profiling_hlib.compiled.hi, } - library_artifacts = { - True: lib, - False: non_profiling_hlib.libs[0], + object_artifacts = { + True: compiled.objects, + False: non_profiling_hlib.compiled.objects, } all_libs = libs + non_profiling_hlib.libs stub_dirs = [compiled.stubs] + [non_profiling_hlib.compiled.stubs] else: + dynamic = { + False: compiled.module_tsets, + } import_artifacts = { False: compiled.hi, } - library_artifacts = { - False: lib, + object_artifacts = { + False: compiled.objects, } all_libs = libs stub_dirs = [compiled.stubs] @@ -645,21 +742,51 @@ def _build_haskell_lib( pkgname, libstem, uniq_infos, - import_artifacts, - library_artifacts, + import_artifacts.keys(), + enable_profiling = enable_profiling, + use_empty_lib = False, + md_file = md_file, + ) + empty_db = _make_package( + ctx, + link_style, + pkgname, + None, + uniq_infos, + import_artifacts.keys(), enable_profiling = enable_profiling, + use_empty_lib = True, + md_file = md_file, ) + deps_db = _make_package( + ctx, + link_style, + pkgname, + None, + uniq_infos, + import_artifacts.keys(), + enable_profiling = enable_profiling, + use_empty_lib = True, + md_file = md_file, + for_deps = True, + ) + hlib = HaskellLibraryInfo( name = pkgname, db = db, + empty_db = empty_db, + deps_db = deps_db, id = pkgname, + dynamic = dynamic, # TODO(ah) refine with dynamic projections import_dirs = import_artifacts, + objects = object_artifacts, stub_dirs = stub_dirs, libs = all_libs, version = "1.0.0", is_prebuilt = False, profiling_enabled = enable_profiling, + dependencies = toolchain_libs, ) return HaskellLibBuildOutput( @@ -694,6 +821,11 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: libname = repr(ctx.label.path).replace("//", "_").replace("/", "_") + "_" + ctx.label.name pkgname = libname.replace("_", "-") + md_file = target_metadata( + ctx, + sources = ctx.attrs.srcs, + ) + # The non-profiling library is also needed to build the package with # profiling enabled, so we need to keep track of it for each link style. non_profiling_hlib = {} @@ -712,6 +844,9 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: nlis = nlis, link_style = link_style, enable_profiling = enable_profiling, + # enable haddock only for the first non-profiling hlib + enable_haddock = not enable_profiling and not non_profiling_hlib, + md_file = md_file, non_profiling_hlib = non_profiling_hlib.get(link_style), ) if not enable_profiling: @@ -754,6 +889,11 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets[link_style.value.replace("_", "-")] = [DefaultInfo( default_outputs = libs, + sub_targets = _haskell_module_sub_targets( + compiled = compiled, + link_style = link_style, + enable_profiling = enable_profiling, + ), )] pic_behavior = ctx.attrs._cxx_toolchain[CxxToolchainInfo].pic_behavior @@ -830,6 +970,38 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: # )] pp = [] + haddock, = haskell_haddock_lib( + ctx, + pkgname, + non_profiling_hlib[LinkStyle("shared")].compiled, + md_file, + ), + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + + styles = [ + ctx.actions.declare_output("haddock-html", file) + for file in "synopsis.png linuwial.css quick-jump.css haddock-bundle.min.js".split() + ] + ctx.actions.run( + cmd_args( + haskell_toolchain.haddock, + "--gen-index", + "-o", cmd_args(styles[0].as_output(), parent=1), + hidden=[file.as_output() for file in styles] + ), + category = "haddock_styles", + ) + sub_targets.update({ + "haddock": [DefaultInfo( + default_outputs = haddock.html.values(), + sub_targets = { + module: [DefaultInfo(default_output = html, other_outputs=styles)] + for module, html in haddock.html.items() + } + )] + }) + providers = [ DefaultInfo( default_outputs = default_output, @@ -854,7 +1026,7 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: shared_libs, shared_library_infos, ), - haskell_haddock_lib(ctx, pkgname), + haddock, ] if indexing_tsets: @@ -896,7 +1068,7 @@ def haskell_library_impl(ctx: AnalysisContext) -> list[Provider]: def derive_indexing_tset( actions: AnalysisActions, link_style: LinkStyle, - value: Artifact | None, + value: list[Artifact] | None, children: list[Dependency]) -> HaskellIndexingTSet: index_children = [] for dep in children: @@ -911,6 +1083,88 @@ def derive_indexing_tset( children = index_children, ) +def _make_link_package( + ctx: AnalysisContext, + link_style: LinkStyle, + pkgname: str, + hlis: list[HaskellLibraryInfo], + static_libs: ArgLike) -> Artifact: + artifact_suffix = get_artifact_suffix(link_style, False) + + conf = cmd_args( + "name: " + pkgname, + "version: 1.0.0", + "id: " + pkgname, + "key: " + pkgname, + "exposed: False", + cmd_args(cmd_args(static_libs, delimiter = ", "), format = "ld-options: {}"), + "depends: " + ", ".join([lib.id for lib in hlis]), + ) + + pkg_conf = ctx.actions.write("pkg-" + artifact_suffix + "_link.conf", conf) + db = ctx.actions.declare_output("db-" + artifact_suffix + "_link", dir = True) + + # While the list of hlis is unique, there may be multiple packages in the same db. + # Cutting down the GHC_PACKAGE_PATH significantly speeds up GHC. + db_deps = {x.db: None for x in hlis}.keys() + + # So that ghc-pkg can find the DBs for the dependencies. We might + # be able to use flags for this instead, but this works. + ghc_package_path = cmd_args( + db_deps, + delimiter = ":", + ) + + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + ctx.actions.run( + cmd_args([ + "sh", + "-c", + _REGISTER_PACKAGE, + "", + haskell_toolchain.packager, + db.as_output(), + pkg_conf, + ]), + category = "haskell_package_link" + artifact_suffix.replace("-", "_"), + env = {"GHC_PACKAGE_PATH": ghc_package_path}, + ) + + return db + +def _dynamic_link_binary_impl(actions, artifacts, dynamic_values, outputs, arg): + link_cmd = arg.link.copy() # link is already frozen, make a copy + + # Add -package-db and -package/-expose-package flags for each Haskell + # library dependency. + packages_info = get_packages_info2( + actions, + deps = arg.deps, + direct_deps_link_info = arg.direct_deps_link_info, + haskell_toolchain = arg.haskell_toolchain, + haskell_direct_deps_lib_infos = arg.haskell_direct_deps_lib_infos, + link_style = arg.link_style, + resolved = dynamic_values, + specify_pkg_version = False, + enable_profiling = arg.enable_profiling, + use_empty_lib = False, + ) + + link_cmd.add("-hide-all-packages") + link_cmd.add(cmd_args(arg.toolchain_libs, prepend = "-package")) + link_cmd.add(cmd_args(packages_info.exposed_package_args)) + link_cmd.add(cmd_args(packages_info.packagedb_args, prepend = "-package-db")) + link_cmd.add(arg.haskell_toolchain.linker_flags) + link_cmd.add(arg.linker_flags) + + link_cmd.add("-o", outputs[arg.output].as_output()) + + actions.run(link_cmd, category = "haskell_link") + + return [] + +_dynamic_link_binary = dynamic_actions(impl = _dynamic_link_binary_impl) + def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: enable_profiling = ctx.attrs.enable_profiling @@ -925,29 +1179,33 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: if enable_profiling and link_style == LinkStyle("shared"): link_style = LinkStyle("static") + md_file = target_metadata(ctx, sources = ctx.attrs.srcs) + compiled = compile( ctx, link_style, enable_profiling = enable_profiling, + enable_haddock = False, + md_file = md_file, ) haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + toolchain_libs = [dep[HaskellToolchainLibrary].name for dep in ctx.attrs.deps if HaskellToolchainLibrary in dep] + output = ctx.actions.declare_output(ctx.attrs.name) - link = cmd_args( - [haskell_toolchain.compiler] + - ["-o", output.as_output()] + - [haskell_toolchain.linker_flags] + - [ctx.attrs.linker_flags], - hidden = compiled.stubs, - ) + link = cmd_args(haskell_toolchain.compiler) - link_args = cmd_args() + objects = {} - osuf, _hisuf = output_extensions(link_style, enable_profiling) + # only add the first object per module + # TODO[CB] restructure this to use a record / dict for compiled.objects + for obj in compiled.objects: + key = paths.replace_extension(obj.short_path, "") + if not key in objects: + objects[key] = obj - objfiles = _srcs_to_objfiles(ctx, compiled.objects, osuf) - link_args.add(objfiles) + link.add(objects.values()) indexing_tsets = {} if compiled.producing_indices: @@ -1095,15 +1353,54 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: sos.extend(traverse_shared_library_info(shlib_info)) infos = get_link_args_for_strategy(ctx, nlis, to_link_strategy(link_style)) - link_args.add(cmd_args(unpack_link_args(infos), prepend = "-optl")) + if link_style in [LinkStyle("static"), LinkStyle("static_pic")]: + hlis = attr_deps_haskell_link_infos_sans_template_deps(ctx) + linfos = [x.prof_info if enable_profiling else x.info for x in hlis] + uniq_infos = [x[link_style].value for x in linfos] + + pkgname = ctx.label.name + "-link" + linkable_artifacts = [ + f.archive.artifact + for link in infos.tset.infos.traverse(ordering = "topological") + for f in link.default.linkables + ] + db = _make_link_package( + ctx, + link_style, + pkgname, + uniq_infos, + linkable_artifacts, + ) + + link.add(cmd_args(db, prepend = "-package-db")) + link.add("-package", pkgname) + link.add(cmd_args(hidden = linkable_artifacts)) + else: + link.add(cmd_args(unpack_link_args(infos), prepend = "-optl")) + + haskell_direct_deps_lib_infos = attr_deps_haskell_lib_infos( + ctx, + link_style, + enable_profiling = enable_profiling, + ) - link.add(at_argfile( - actions = ctx.actions, - name = "args.haskell_link_argsfile", - args = link_args, - allow_args = True, + ctx.actions.dynamic_output_new(_dynamic_link_binary( + dynamic = [], + dynamic_values = [haskell_toolchain.packages.dynamic] if haskell_toolchain.packages else [ ], + outputs = [output.as_output()], + arg = struct( + deps = ctx.attrs.deps, + direct_deps_link_info = attr_deps_haskell_link_infos(ctx), + enable_profiling = enable_profiling, + haskell_direct_deps_lib_infos = haskell_direct_deps_lib_infos, + haskell_toolchain = haskell_toolchain, + link = link, + link_style = link_style, + linker_flags = ctx.attrs.linker_flags, + output = output, + toolchain_libs = toolchain_libs, + ), )) - ctx.actions.run(link, category = "haskell_link") if link_style == LinkStyle("shared") or link_group_info != None: sos_dir = "__{}__shared_libs_symlink_tree".format(ctx.attrs.name) @@ -1119,8 +1416,18 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: else: run = cmd_args(output) + sub_targets = {} + sub_targets.update(_haskell_module_sub_targets( + compiled = compiled, + link_style = link_style, + enable_profiling = enable_profiling, + )) + providers = [ - DefaultInfo(default_output = output), + DefaultInfo( + default_output = output, + sub_targets = sub_targets, + ), RunInfo(args = run), ] @@ -1128,3 +1435,18 @@ def haskell_binary_impl(ctx: AnalysisContext) -> list[Provider]: providers.append(HaskellIndexInfo(info = indexing_tsets)) return providers + +def _haskell_module_sub_targets(*, compiled, link_style, enable_profiling): + (osuf, hisuf) = output_extensions(link_style, enable_profiling) + return { + "interfaces": [DefaultInfo(sub_targets = { + src_to_module_name(hi.short_path): [DefaultInfo(default_output = hi)] + for hi in compiled.hi + if hi.extension[1:] == hisuf + })], + "objects": [DefaultInfo(sub_targets = { + src_to_module_name(o.short_path): [DefaultInfo(default_output = o)] + for o in compiled.objects + if o.extension[1:] == osuf + })], + } diff --git a/haskell/haskell_ghci.bzl b/haskell/haskell_ghci.bzl index a59fbd121..790ca3c52 100644 --- a/haskell/haskell_ghci.bzl +++ b/haskell/haskell_ghci.bzl @@ -636,6 +636,7 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: link_style, specify_pkg_version = True, enable_profiling = enable_profiling, + use_empty_lib = False, ) # Create package db symlinks @@ -660,7 +661,8 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: for prof, import_dir in lib.import_dirs.items(): artifact_suffix = get_artifact_suffix(link_style, prof) - lib_symlinks["hi-" + artifact_suffix] = import_dir + for imp in import_dir: + lib_symlinks["mod-" + artifact_suffix + "/" + imp.short_path] = imp for o in lib.libs: lib_symlinks[o.short_path] = o @@ -729,7 +731,9 @@ def haskell_ghci_impl(ctx: AnalysisContext) -> list[Provider]: "__{}__".format(ctx.label.name), output_artifacts, ) - run = cmd_args(final_ghci_script, hidden = outputs) + ghci_bin_dep = ctx.attrs.ghci_bin_dep.get(RunInfo) + hidden_dep = [ghci_bin_dep] if ghci_bin_dep else [] + run = cmd_args(final_ghci_script, hidden=hidden_dep + outputs) return [ DefaultInfo(default_outputs = [root_output_dir]), diff --git a/haskell/haskell_haddock.bzl b/haskell/haskell_haddock.bzl index 4154e3aba..ec836dca3 100644 --- a/haskell/haskell_haddock.bzl +++ b/haskell/haskell_haddock.bzl @@ -14,99 +14,204 @@ load( load( "@prelude//haskell:util.bzl", "attr_deps", + "attr_deps_haskell_link_infos", + "src_to_module_name", ) -load("@prelude//utils:argfile.bzl", "at_argfile") +load("@prelude//utils:graph_utils.bzl", "post_order_traversal") +load("@prelude//:paths.bzl", "paths") HaskellHaddockInfo = provider( fields = { - "html": provider_field(typing.Any, default = None), - "interface": provider_field(typing.Any, default = None), + "html": provider_field(dict[str, typing.Any], default = {}), + "interfaces": provider_field(list[typing.Any], default = []), }, ) -def haskell_haddock_lib(ctx: AnalysisContext, pkgname: str) -> Provider: - haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] - iface = ctx.actions.declare_output("haddock-interface") - odir = ctx.actions.declare_output("haddock-html", dir = True) +_HaddockInfo = record( + interface = Artifact, + haddock = Artifact, + html = Artifact, +) + +def _haskell_interfaces_args(info: _HaddockInfo): + return cmd_args(info.interface, format="--one-shot-dep-hi={}") - link_style = cxx_toolchain_link_style(ctx) - args = compile_args( - ctx, - link_style, - enable_profiling = False, - suffix = "-haddock", - pkgname = pkgname, +_HaddockInfoTSet = transitive_set( + args_projections = { + "interfaces": _haskell_interfaces_args + } +) + +def _haddock_module_to_html(module_name: str) -> str: + return module_name.replace(".", "-") + ".html" + +def _haddock_dump_interface( + actions: AnalysisActions, + cmd: cmd_args, + module_name: str, + module_tsets: dict[str, _HaddockInfoTSet], + haddock_info: _HaddockInfo, + module_deps: list[CompiledModuleTSet], + graph: dict[str, list[str]], + outputs: dict[Artifact, Artifact]) -> _HaddockInfoTSet: + + # Transitive module dependencies from other packages. + cross_package_modules = actions.tset( + CompiledModuleTSet, + children = module_deps, + ) + cross_interfaces = cross_package_modules.project_as_args("interfaces") + + # Transitive module dependencies from the same package. + this_package_modules = [ + module_tsets[dep_name] + for dep_name in graph[module_name] + ] + + expected_html = outputs[haddock_info.html] + module_html = _haddock_module_to_html(module_name) + + if paths.basename(expected_html.short_path) != module_html: + html_output = actions.declare_output("haddock-html", module_html) + make_copy = True + else: + html_output = expected_html + make_copy = False + + actions.run( + cmd.copy().add( + "--odir", cmd_args(html_output.as_output(), parent = 1), + "--dump-interface", outputs[haddock_info.haddock].as_output(), + "--html", + "--hoogle", + cmd_args( + haddock_info.interface, + format="--one-shot-hi={}"), + cmd_args( + [haddock_info.project_as_args("interfaces") for haddock_info in this_package_modules], + ), + cmd_args( + cross_interfaces, format="--one-shot-dep-hi={}" + ) + ), + category = "haskell_haddock", + identifier = module_name, + no_outputs_cleanup = True, + ) + if make_copy: + # XXX might as well use `symlink_file`` but that does not work with buck2 RE + # (see https://github.com/facebook/buck2/issues/222) + actions.copy_file(expected_html.as_output(), html_output) + + return actions.tset( + _HaddockInfoTSet, + value = _HaddockInfo(interface = haddock_info.interface, haddock = outputs[haddock_info.haddock], html = outputs[haddock_info.html]), + children = this_package_modules, ) +def _dynamic_haddock_dump_interfaces_impl(actions, artifacts, dynamic_values, outputs, arg): + md = artifacts[arg.md_file].read_json() + module_map = md["module_mapping"] + graph = md["module_graph"] + package_deps = md["package_deps"] + + dynamic_info_lib = {} + + for lib in arg.direct_deps_link_info: + info = lib.info[arg.link_style] + direct = info.value + dynamic = direct.dynamic[False] + dynamic_info = dynamic_values[dynamic].providers[DynamicCompileResultInfo] + + dynamic_info_lib[direct.name] = dynamic_info + + haddock_infos = { module_map.get(k, k): v for k, v in arg.haddock_infos.items() } + module_tsets = {} + + for module_name in post_order_traversal(graph): + module_deps = [ + info.modules[mod] + for lib, info in dynamic_info_lib.items() + for mod in package_deps.get(module_name, {}).get(lib, []) + ] + + module_tsets[module_name] = _haddock_dump_interface( + actions, + arg.dyn_cmd.copy(), + module_name = module_name, + module_tsets = module_tsets, + haddock_info = haddock_infos[module_name], + module_deps = module_deps, + graph = graph, + outputs = outputs, + ) + + return [] + +_dynamic_haddock_dump_interfaces = dynamic_actions(impl = _dynamic_haddock_dump_interfaces_impl) + +def haskell_haddock_lib(ctx: AnalysisContext, pkgname: str, compiled: CompileResultInfo, md_file: Artifact) -> HaskellHaddockInfo: + haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] + + link_style = cxx_toolchain_link_style(ctx) + cmd = cmd_args(haskell_toolchain.haddock) - cmd.add(cmd_args(args.args_for_cmd, format = "--optghc={}")) + cmd.add( "--use-index", "doc-index.html", "--use-contents", "index.html", - "--html", - "--hoogle", "--no-tmp-comp-dir", "--no-warnings", - "--dump-interface", - iface.as_output(), - "--odir", - odir.as_output(), "--package-name", pkgname, ) - for lib in attr_deps(ctx): - hi = lib.get(HaskellHaddockInfo) - if hi != None: - cmd.add("--read-interface", hi.interface) - cmd.add(ctx.attrs.haddock_flags) source_entity = read_root_config("haskell", "haddock_source_entity", None) if source_entity: cmd.add("--source-entity", source_entity) - if args.args_for_file: - if haskell_toolchain.use_argsfile: - ghcargs = cmd_args(args.args_for_file, format = "--optghc={}") - cmd.add(at_argfile( - actions = ctx.actions, - name = "args.haskell_haddock_argsfile", - args = [ghcargs, args.srcs], - allow_args = True, - )) - else: - cmd.add(args.args_for_file) - - # Buck2 requires that the output artifacts are always produced, but Haddock only - # creates them if it needs to, so we need a wrapper script to mkdir the outputs. - script = ctx.actions.declare_output("haddock-script") - script_args = cmd_args([ - "mkdir", - "-p", - args.result.objects.as_output(), - args.result.hi.as_output(), - args.result.stubs.as_output(), - "&&", - cmd_args(cmd, quote = "shell"), - ], delimiter = " ") - ctx.actions.write( - script, - cmd_args("#!/bin/sh", script_args), - is_executable = True, - allow_args = True, - ) + haddock_infos = { + src_to_module_name(hi.short_path): _HaddockInfo( + interface = hi, + haddock = ctx.actions.declare_output("haddock-interface/{}.haddock".format(src_to_module_name(hi.short_path))), + html = ctx.actions.declare_output("haddock-html", _haddock_module_to_html(src_to_module_name(hi.short_path))), + ) + for hi in compiled.hi + if not hi.extension.endswith("-boot") + } - ctx.actions.run( - cmd_args(script, hidden = cmd), - category = "haskell_haddock", - no_outputs_cleanup = True, - ) + direct_deps_link_info = attr_deps_haskell_link_infos(ctx) + + ctx.actions.dynamic_output_new(_dynamic_haddock_dump_interfaces( + dynamic = [md_file], + dynamic_values = [ + info.value.dynamic[False] + for lib in direct_deps_link_info + for info in [ + #lib.prof_info[link_style] + #if enable_profiling else + lib.info[link_style], + ] + ], + outputs = [output.as_output() for info in haddock_infos.values() for output in [info.haddock, info.html]], + arg = struct( + direct_deps_link_info = direct_deps_link_info, + dyn_cmd = cmd.copy(), + haddock_infos = haddock_infos, + link_style = link_style, + md_file = md_file, + ), + )) - return HaskellHaddockInfo(interface = iface, html = odir) + return HaskellHaddockInfo( + interfaces = [i.haddock for i in haddock_infos.values()], + html = {module: i.html for module, i in haddock_infos.items()}, + ) def haskell_haddock_impl(ctx: AnalysisContext) -> list[Provider]: haskell_toolchain = ctx.attrs._haskell_toolchain[HaskellToolchainInfo] diff --git a/haskell/library_info.bzl b/haskell/library_info.bzl index 3b048f137..5eb033a84 100644 --- a/haskell/library_info.bzl +++ b/haskell/library_info.bzl @@ -5,6 +5,8 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//utils:utils.bzl", "flatten", "dedupe_by_value") + # If the target is a haskell library, the HaskellLibraryProvider # contains its HaskellLibraryInfo. (in contrast to a HaskellLinkInfo, # which contains the HaskellLibraryInfo for all the transitive @@ -23,10 +25,18 @@ HaskellLibraryInfo = record( name = str, # package config database: e.g. platform009/build/ghc/lib/package.conf.d db = Artifact, + # package config database, referring to the empty lib which is only used for compilation + empty_db = Artifact, + # pacakge config database, used for ghc -M + deps_db = Artifact, # e.g. "base-4.13.0.0" id = str, + # dynamic dependency information + dynamic = None | dict[bool, DynamicValue], # Import dirs indexed by profiling enabled/disabled - import_dirs = dict[bool, Artifact], + import_dirs = dict[bool, list[Artifact]], + # Object files indexed by profiling enabled/disabled + objects = dict[bool, list[Artifact]], stub_dirs = list[Artifact], # This field is only used as hidden inputs to compilation, to @@ -40,6 +50,32 @@ HaskellLibraryInfo = record( version = str, is_prebuilt = bool, profiling_enabled = bool, + # Package dependencies + dependencies = list[str], ) -HaskellLibraryInfoTSet = transitive_set() +def _project_as_package_db(lib: HaskellLibraryInfo): + return cmd_args(lib.db) + +def _project_as_empty_package_db(lib: HaskellLibraryInfo): + return cmd_args(lib.empty_db) + +def _project_as_deps_package_db(lib: HaskellLibraryInfo): + return cmd_args(lib.deps_db) + +def _get_package_deps(children: list[list[str]], lib: HaskellLibraryInfo | None): + flatted = flatten(children) + if lib: + flatted.extend(lib.dependencies) + return dedupe_by_value(flatted) + +HaskellLibraryInfoTSet = transitive_set( + args_projections = { + "package_db": _project_as_package_db, + "empty_package_db": _project_as_empty_package_db, + "deps_package_db": _project_as_deps_package_db, + }, + reductions = { + "packages": _get_package_deps, + }, +) diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index fbd140ff7..f57d6d161 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -5,6 +5,8 @@ # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. +load("@prelude//utils:arglike.bzl", "ArgLike") + HaskellPlatformInfo = provider(fields = { "name": provider_field(typing.Any, default = None), }) @@ -37,6 +39,7 @@ HaskellToolchainInfo = provider( "ghci_packager": provider_field(typing.Any, default = None), "cache_links": provider_field(typing.Any, default = None), "script_template_processor": provider_field(typing.Any, default = None), + "packages": provider_field(typing.Any, default = None), }, ) @@ -45,3 +48,25 @@ HaskellToolchainLibrary = provider( "name": provider_field(str), }, ) + +HaskellPackagesInfo = record( + dynamic = DynamicValue, +) + +HaskellPackage = record( + db = ArgLike, + path = Artifact, +) + +def _haskell_package_info_as_package_db(p: HaskellPackage): + return cmd_args(p.db) + +HaskellPackageDbTSet = transitive_set( + args_projections = { + "package_db": _haskell_package_info_as_package_db, + } +) + +DynamicHaskellPackageDbInfo = provider(fields = { + "packages": dict[str, HaskellPackageDbTSet], +})