From 72c320391ab945c0d6703b69bcd5677719461124 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 10 Nov 2023 07:47:35 +0100 Subject: [PATCH 1/9] CI: Add GHC 9.6.2 to "Build & Test - Nixpkgs" job matrix --- .github/workflows/workflow.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 96d017ffe..e6f27d40f 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -55,6 +55,7 @@ jobs: ghc: - 9.2.8 - 9.4.6 + - 9.6.2 exclude: - module: rules_haskell_nix bzlmod: false @@ -62,6 +63,8 @@ jobs: # and stack config per GHC version - ghc: 9.4.6 bzlmod: true + - ghc: 9.6.2 + bzlmod: true runs-on: ${{ matrix.os }} steps: - if: ${{ matrix.os == 'ubuntu-latest' }} From 9d4be3cc31ac3f4b4b12ce196ec008d27feae143 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 14 Jul 2023 17:37:20 +0200 Subject: [PATCH 2/9] Handle different top level directories for nixpkgs GHC Some newer GHC versions have the `package.conf.d` below a `lib` directory. --- haskell/private/pkgdb_to_bzl.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/haskell/private/pkgdb_to_bzl.py b/haskell/private/pkgdb_to_bzl.py index bed48312b..e24a2d6ad 100755 --- a/haskell/private/pkgdb_to_bzl.py +++ b/haskell/private/pkgdb_to_bzl.py @@ -21,6 +21,14 @@ if len(sys.argv) == 3: repo_dir = "external/" + sys.argv[1] topdir = sys.argv[2] + + if os.path.exists(os.path.join(topdir, 'package.conf.d')): + package_conf_dir = os.path.join(topdir, 'package.conf.d') + elif os.path.exists(os.path.join(topdir, 'lib', 'package.conf.d')): + topdir = os.path.join(topdir, 'lib') + package_conf_dir = os.path.join(topdir, 'package.conf.d') + else: + sys.exit("could not find package.conf.d directory at {}".format(topdir)) else: sys.exit("Usage: pkgdb_to_bzl.py ") @@ -79,7 +87,8 @@ def hs_library_pattern(name, mode = "static", profiling = False): # Accumulate package id to package name mappings. pkg_id_map = [] -for conf in glob.glob(os.path.join(topdir, "package.conf.d", "*.conf")): + +for conf in glob.glob(os.path.join(package_conf_dir, '*.conf')): with open(conf, 'r') as f: pkg = package_configuration.parse_package_configuration(f) From 6fcc564d30f14d54388cc895f2e977aec2724533 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 14 Jul 2023 19:18:15 +0200 Subject: [PATCH 3/9] Handle `${pkgroot}` results in relative paths outside of repository --- haskell/private/pkgdb_to_bzl.py | 36 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/haskell/private/pkgdb_to_bzl.py b/haskell/private/pkgdb_to_bzl.py index e24a2d6ad..7642e1cd2 100755 --- a/haskell/private/pkgdb_to_bzl.py +++ b/haskell/private/pkgdb_to_bzl.py @@ -29,13 +29,27 @@ package_conf_dir = os.path.join(topdir, 'package.conf.d') else: sys.exit("could not find package.conf.d directory at {}".format(topdir)) + repo_root = os.getcwd() else: sys.exit("Usage: pkgdb_to_bzl.py ") +def resolve(path, pkgroot): + """Resolve references to ${pkgroot} with the given value""" + if path.find("${pkgroot}") != -1: + return path.strip("\"").replace("${pkgroot}", pkgroot) + else: + return path + def path_to_label(path, pkgroot): """Substitute one pkgroot for another relative one to obtain a label.""" if path.find("${pkgroot}") != -1: - return os.path.normpath(path.strip("\"").replace("${pkgroot}", topdir)).replace('\\', '/') + # determine if the given path is inside the repository root + # if it is not, return None to signal it needs to be symlinked into the + # repository + real_path = os.path.realpath(resolve(path, pkgroot)) + relative_path = os.path.relpath(real_path, start=repo_root) + + return None if relative_path.startswith('..') else relative_path.replace('\\', '/') topdir_relative_path = path.replace(pkgroot, "$topdir") if topdir_relative_path.find("$topdir") != -1: @@ -114,23 +128,27 @@ def hs_library_pattern(name, mode = "static", profiling = False): haddock_html = None if pkg.haddock_html: - haddock_html = path_to_label(pkg.haddock_html, pkgroot) # We check if the file exists because cabal will unconditionally # generate the database entry even if no haddock was generated. - if not haddock_html and os.path.exists(pkg.haddock_html): - haddock_html = os.path.join("haddock", "html", pkg.name) - output.append("#SYMLINK: {} {}".format(pkg.haddock_html, haddock_html)) + resolved_haddock_html = resolve(pkg.haddock_html, pkgroot) + + if os.path.exists(resolved_haddock_html): + haddock_html = path_to_label(pkg.haddock_html, pkgroot) + if not haddock_html: + haddock_html = os.path.join("haddock", "html", pkg.name) + output.append("#SYMLINK: {} {}".format(resolved_haddock_html.replace('\\', '/'), haddock_html)) # If there is many interfaces, we give them a number interface_id = 0 haddock_interfaces = [] for interface_path in pkg.haddock_interfaces: - interface = path_to_label(interface_path, pkgroot) + resolved_path = resolve(interface_path, pkgroot).replace('\\', '/') # We check if the file exists because cabal will unconditionally # generate the database entry even if no haddock was generated. - if not os.path.exists(interface or interface_path): - continue + if not os.path.exists(resolved_path): continue + + interface = path_to_label(interface_path, pkgroot) if not interface: interface = os.path.join( @@ -138,7 +156,7 @@ def hs_library_pattern(name, mode = "static", profiling = False): "interfaces", pkg.name + "_" + str(interface_id) + ".haddock", ) - output.append("#SYMLINK: {} {}".format(interface_path, interface)) + output.append("#SYMLINK: {} {}".format(resolved_path, interface)) interface_id += 1 haddock_interfaces.append(interface) From ac05cc8e9ba5aefae06c78a40b1da1801d6f98a2 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 4 Aug 2023 11:57:27 +0200 Subject: [PATCH 4/9] Resolve $topdir in ghc-pkg paths --- haskell/private/pkgdb_to_bzl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/haskell/private/pkgdb_to_bzl.py b/haskell/private/pkgdb_to_bzl.py index 7642e1cd2..47d04dcf5 100755 --- a/haskell/private/pkgdb_to_bzl.py +++ b/haskell/private/pkgdb_to_bzl.py @@ -34,9 +34,11 @@ sys.exit("Usage: pkgdb_to_bzl.py ") def resolve(path, pkgroot): - """Resolve references to ${pkgroot} with the given value""" + """Resolve references to ${pkgroot} with the given value, resolve $topdir with `topdir`""" if path.find("${pkgroot}") != -1: return path.strip("\"").replace("${pkgroot}", pkgroot) + elif path.startswith("$topdir"): + return os.path.normpath(path.replace("$topdir", topdir)).replace('\\', '/') else: return path From a2533f670a069fefd6f31c5752fcd67b9b9f326f Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 4 Aug 2023 15:47:16 +0200 Subject: [PATCH 5/9] Work around wrong haddock-html path on Windows See https://gitlab.haskell.org/ghc/ghc/-/issues/23476 --- haskell/private/pkgdb_to_bzl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/haskell/private/pkgdb_to_bzl.py b/haskell/private/pkgdb_to_bzl.py index 47d04dcf5..3ee6182ab 100755 --- a/haskell/private/pkgdb_to_bzl.py +++ b/haskell/private/pkgdb_to_bzl.py @@ -134,6 +134,11 @@ def hs_library_pattern(name, mode = "static", profiling = False): # generate the database entry even if no haddock was generated. resolved_haddock_html = resolve(pkg.haddock_html, pkgroot) + if not os.path.exists(resolved_haddock_html): + # try to resolve relative to the package.conf.d dir + # see https://gitlab.haskell.org/ghc/ghc/-/issues/23476 + resolved_haddock_html = resolve(pkg.haddock_html, package_conf_dir) + if os.path.exists(resolved_haddock_html): haddock_html = path_to_label(pkg.haddock_html, pkgroot) if not haddock_html: @@ -146,6 +151,11 @@ def hs_library_pattern(name, mode = "static", profiling = False): for interface_path in pkg.haddock_interfaces: resolved_path = resolve(interface_path, pkgroot).replace('\\', '/') + if not os.path.exists(resolved_path): + # try to resolve relative to the package.conf.d dir + # see https://gitlab.haskell.org/ghc/ghc/-/issues/23476 + resolved_path = resolve(interface_path, package_conf_dir) + # We check if the file exists because cabal will unconditionally # generate the database entry even if no haddock was generated. if not os.path.exists(resolved_path): continue From bc8d408603d7189de5ea48554a0d354690ab736f Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 10 Nov 2023 07:27:27 +0100 Subject: [PATCH 6/9] Remove `haddock/rts-1.0.2` from expected output for test-haddock ... on Windows. The pkg conf file for rts has entries for `haddock-interface` and `haddock-html` but neither of those actually exist. --- rules_haskell_tests/tests/BUILD.bazel | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rules_haskell_tests/tests/BUILD.bazel b/rules_haskell_tests/tests/BUILD.bazel index f74d4c879..c110160db 100644 --- a/rules_haskell_tests/tests/BUILD.bazel +++ b/rules_haskell_tests/tests/BUILD.bazel @@ -248,9 +248,7 @@ rule_test( "haddock/testsZShaddockZShaddock-lib-b", "haddock/testsZShaddockZShaddock-lib-deep", ], - }[TEST_GHC_VERSION] + ([ - "haddock/rts-1.0.2", - ] if is_windows else []), + }[TEST_GHC_VERSION], rule = "//tests/haddock", ) From b65ebd80e6671533ec82809a4af9de4a8264ba94 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 1 Dec 2023 10:56:50 +0100 Subject: [PATCH 7/9] Construct repository relative paths for absolute and relative paths --- haskell/private/pkgdb_to_bzl.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/haskell/private/pkgdb_to_bzl.py b/haskell/private/pkgdb_to_bzl.py index 3ee6182ab..926550807 100755 --- a/haskell/private/pkgdb_to_bzl.py +++ b/haskell/private/pkgdb_to_bzl.py @@ -36,7 +36,11 @@ def resolve(path, pkgroot): """Resolve references to ${pkgroot} with the given value, resolve $topdir with `topdir`""" if path.find("${pkgroot}") != -1: - return path.strip("\"").replace("${pkgroot}", pkgroot) + norm_path = os.path.normpath(path.strip("\"").replace("${pkgroot}", pkgroot)) + if not os.path.isabs(norm_path) and norm_path.startswith('..'): + return resolve(path, os.path.realpath(pkgroot)) + else: + return norm_path elif path.startswith("$topdir"): return os.path.normpath(path.replace("$topdir", topdir)).replace('\\', '/') else: @@ -48,12 +52,12 @@ def path_to_label(path, pkgroot): # determine if the given path is inside the repository root # if it is not, return None to signal it needs to be symlinked into the # repository - real_path = os.path.realpath(resolve(path, pkgroot)) - relative_path = os.path.relpath(real_path, start=repo_root) + norm_path = os.path.normpath(resolve(path, pkgroot)) + relative_path = os.path.relpath(norm_path, start=repo_root) return None if relative_path.startswith('..') else relative_path.replace('\\', '/') - topdir_relative_path = path.replace(pkgroot, "$topdir") + topdir_relative_path = path.replace(os.path.realpath(pkgroot), "$topdir") if topdir_relative_path.find("$topdir") != -1: return os.path.normpath(topdir_relative_path.replace("$topdir", topdir)).replace('\\', '/') @@ -104,15 +108,16 @@ def hs_library_pattern(name, mode = "static", profiling = False): # Accumulate package id to package name mappings. pkg_id_map = [] +# pkgroot is not part of .conf files. It's a computed value. It is +# defined to be the directory enclosing the package database +# directory. +pkgroot = os.path.dirname(package_conf_dir) + + for conf in glob.glob(os.path.join(package_conf_dir, '*.conf')): with open(conf, 'r') as f: pkg = package_configuration.parse_package_configuration(f) - # pkgroot is not part of .conf files. It's a computed value. It is - # defined to be the directory enclosing the package database - # directory. - pkgroot = os.path.dirname(os.path.dirname(os.path.realpath(conf))) - pkg_id_map.append((pkg.name, pkg.id)) # Haddock handling From 745663e86c175ca5a85532f4161241d0526324b8 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Thu, 16 Nov 2023 09:07:19 +0100 Subject: [PATCH 8/9] Work around `otool` not set to nix store path in GHC settings In previous GHC versions from nixpkgs, the `otool` setting was referencing a tool in the nix store, but for GHC 9.6.2 it is just set to "otool" which means it must be in `$PATH`. The same applies to the `install_name_tool`. See https://github.com/NixOS/nixpkgs/issues/267250 and https://gitlab.haskell.org/ghc/ghc/-/issues/24211 We work around by using the location of the `ar` command and assume the other tools (from the bintools package) are also available at the same place. --- haskell/cabal.bzl | 9 +++++++++ haskell/experimental/private/module.bzl | 7 +++++++ haskell/private/actions/compile.bzl | 9 +++++++++ haskell/private/actions/link.bzl | 16 ++++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/haskell/cabal.bzl b/haskell/cabal.bzl index a935f23e7..081ab0c02 100644 --- a/haskell/cabal.bzl +++ b/haskell/cabal.bzl @@ -289,6 +289,15 @@ def _prepare_cabal_inputs( ] extra_args = ["--flags=" + " ".join(flags)] + if hs.toolchain.is_darwin: + # assume `otool` and `install_name_tool` are available at the same location as `ar` + ar_bindir = paths.dirname(cc.tools.ar) + + extra_args.append("--ghc-option=-pgmotool=" + paths.join(ar_bindir, "otool")) + extra_args.append("--ghc-option=-pgminstall_name_tool=" + paths.join(ar_bindir, "install_name_tool")) + extra_args.append("--haddock-option=--optghc=-pgmotool=" + paths.join(ar_bindir, "otool")) + extra_args.append("--haddock-option=--optghc=-pgminstall_name_tool=" + paths.join(ar_bindir, "install_name_tool")) + ghc_version = [int(x) for x in hs.toolchain.version.split(".")] if dynamic_file: # See Note [No PIE when linking] in haskell/private/actions/link.bzl diff --git a/haskell/experimental/private/module.bzl b/haskell/experimental/private/module.bzl index 8da21d4a2..e9c3e2613 100644 --- a/haskell/experimental/private/module.bzl +++ b/haskell/experimental/private/module.bzl @@ -302,6 +302,13 @@ def _build_haskell_module( args.add_all(hs.toolchain.ghcopts) args.add_all(user_ghcopts) + if hs.toolchain.is_darwin: + # assume `otool` and `install_name_tool` are available at the same location as `ar` + ar_bindir = paths.dirname(cc.tools.ar) + + args.add(paths.join(ar_bindir, "otool"), format = "-pgmotool=%s") + args.add(paths.join(ar_bindir, "install_name_tool"), format = "-pgminstall_name_tool=%s") + if plugins and not enable_th: # For #1681. These suppresses bogus warnings about missing libraries which # aren't really needed. diff --git a/haskell/private/actions/compile.bzl b/haskell/private/actions/compile.bzl index ef5ba3dfe..183d51267 100644 --- a/haskell/private/actions/compile.bzl +++ b/haskell/private/actions/compile.bzl @@ -124,6 +124,15 @@ def _compilation_defaults( compile_flags += hs.toolchain.ghcopts compile_flags += user_compile_flags + if hs.toolchain.is_darwin: + # assume `otool` and `install_name_tool` are available at the same location as `ar` + ar_bindir = paths.dirname(cc.tools.ar) + + compile_flags += [ + "-pgmotool=" + paths.join(ar_bindir, "otool"), + "-pgminstall_name_tool=" + paths.join(ar_bindir, "install_name_tool"), + ] + package_ids = [] all_plugins = plugins + non_default_plugins for plugin in all_plugins: diff --git a/haskell/private/actions/link.bzl b/haskell/private/actions/link.bzl index 278314815..b581fc7e3 100644 --- a/haskell/private/actions/link.bzl +++ b/haskell/private/actions/link.bzl @@ -133,6 +133,14 @@ def link_binary( args.add_all(cc.linker_flags, format_each = "-optl%s") if with_profiling: args.add("-prof") + + if hs.toolchain.is_darwin: + # assume `otool` and `install_name_tool` are available at the same location as `ar` + ar_bindir = paths.dirname(cc.tools.ar) + + args.add(paths.join(ar_bindir, "otool"), format = "-pgmotool=%s") + args.add(paths.join(ar_bindir, "install_name_tool"), format = "-pgminstall_name_tool=%s") + args.add_all(hs.toolchain.ghcopts) args.add_all(compiler_flags) @@ -366,6 +374,14 @@ def link_library_dynamic(hs, cc, posix, dep_info, extra_srcs, object_files, my_p args = hs.actions.args() args.add_all(cc.linker_flags, format_each = "-optl%s") args.add_all(["-shared", "-dynamic"]) + + if hs.toolchain.is_darwin: + # assume `otool` and `install_name_tool` are available at the same location as `ar` + ar_bindir = paths.dirname(cc.tools.ar) + + args.add(paths.join(ar_bindir, "otool"), format = "-pgmotool=%s") + args.add(paths.join(ar_bindir, "install_name_tool"), format = "-pgminstall_name_tool=%s") + args.add_all(hs.toolchain.ghcopts) args.add_all(compiler_flags) From ce050b0e81f3b7d1d0dc8650c877b9f09a3cb07e Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Wed, 6 Dec 2023 17:53:46 +0100 Subject: [PATCH 9/9] Skip profiling test on macos with GHC 9.6.2 Trying to build this results in a bus error with ghc-iserv. --- rules_haskell_tests/tests/RunTests.hs | 7 ++++++- .../tests/haskell_module/dep-narrowing-th/BUILD.bazel | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/rules_haskell_tests/tests/RunTests.hs b/rules_haskell_tests/tests/RunTests.hs index 1e036859c..78bdab253 100644 --- a/rules_haskell_tests/tests/RunTests.hs +++ b/rules_haskell_tests/tests/RunTests.hs @@ -10,6 +10,7 @@ import System.Directory (copyFile) import System.FilePath (()) import System.Info (os) import System.IO.Temp (withSystemTempDirectory) +import System.Environment (lookupEnv) import qualified System.Process as Process import Test.Hspec.Core.Spec (SpecM) @@ -26,11 +27,15 @@ main = hspec $ do assertSuccess (bazel ["test", "//..."]) it "bazel test prof" $ do + ghcVersion <- lookupEnv "GHC_VERSION" + -- In .github/workflows/workflow.yaml we specify --test_tag_filters -- -dont_test_on_darwin. However, specifiying --test_tag_filters -- -requires_dynamic here alone would override that filter. So, -- we have to duplicate that filter here. - let tagFilter | os == "darwin" = "-dont_test_on_darwin,-requires_dynamic,-skip_profiling" + let tagFilter | os == "darwin" = "-dont_test_on_darwin,-requires_dynamic,-skip_profiling" ++ ( + -- skip tests for specific GHC version, see https://github.com/tweag/rules_haskell/issues/2073 + maybe "" (",-dont_build_on_macos_with_ghc_" ++) ghcVersion) | otherwise = "-requires_dynamic,-skip_profiling" assertSuccess (bazel ["test", "-c", "dbg", "//...", "--build_tag_filters", tagFilter, "--test_tag_filters", tagFilter]) diff --git a/rules_haskell_tests/tests/haskell_module/dep-narrowing-th/BUILD.bazel b/rules_haskell_tests/tests/haskell_module/dep-narrowing-th/BUILD.bazel index dc306075c..4e18f5484 100644 --- a/rules_haskell_tests/tests/haskell_module/dep-narrowing-th/BUILD.bazel +++ b/rules_haskell_tests/tests/haskell_module/dep-narrowing-th/BUILD.bazel @@ -94,6 +94,11 @@ haskell_library( narrowed_deps = [ ":TestLib2", ], + tags = [ + # skip building this target on macos with GHC 9.6.2 since it crashes ghc-iserv + # see https://github.com/tweag/rules_haskell/issues/2073 + "dont_build_on_macos_with_ghc_9.6.2", + ], deps = [ ":NonModulesTestLib", ":TestLib", @@ -113,6 +118,11 @@ haskell_test( name = "Test", modules = [":TestBinModule"], narrowed_deps = [":lib"], + tags = [ + # skip testing this target on macos with GHC 9.6.2 since it crashes ghc-iserv + # see https://github.com/tweag/rules_haskell/issues/2073 + "dont_build_on_macos_with_ghc_9.6.2", + ], visibility = ["//visibility:public"], deps = [ "//tests/hackage:base",