Merge pull request #1478 from tweag/bazelci-settings

Track GHC's lib/settings as an action input

Author: aherrmann

WORKSPACE

@@ -73,6 +73,8 @@ stack_snapshot(
         "language-c",
         "streaming",
         "void",
+        "ghc-paths",
+        "ghc-check-0.5.0.3",
         "hspec",
         "hspec-core",
         "lens-family-core",

haskell/cabal.bzl

@@ -282,6 +282,7 @@ def _prepare_cabal_inputs(
         transitive = [
             depset(srcs),
             depset(cc.files),
+            depset(hs.toolchain.files),
             package_databases,
             setup_dep_info.package_databases,
             transitive_headers,

haskell/ghc.BUILD.tpl

@@ -15,6 +15,8 @@ load(
 
 package(default_visibility = ["//visibility:public"])
 
+%{toolchain_libraries}
+
 %{toolchain}
 
 filegroup(

haskell/ghc_bindist.bzl

@@ -3,10 +3,20 @@
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "patch")
 load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value")
 load("@rules_sh//sh:posix.bzl", "sh_posix_configure")
-load(":private/workspace_utils.bzl", "execute_or_fail_loudly")
+load(
+    ":private/pkgdb_to_bzl.bzl",
+    "pkgdb_to_bzl",
+)
+load(
+    ":private/workspace_utils.bzl",
+    "define_rule",
+    "execute_or_fail_loudly",
+    "find_python",
+    "resolve_labels",
+)
 
-# If you change this, change stackage's version
-# in the start script (see stackage.org)
+# If you change this, change stackage's version in the start script
+# (see stackage.org).
 _GHC_DEFAULT_VERSION = "8.6.5"
 
 # Generated with `bazel run @rules_haskell//haskell:gen-ghc-bindist`
@@ -271,14 +281,13 @@ GHC_BINDIST = \
     }
 
 def _ghc_bindist_impl(ctx):
-    # Avoid rule restart by resolving these labels early. See
-    # https://github.com/bazelbuild/bazel/blob/master/tools/cpp/lib_cc_configure.bzl#L17.
-    ghc_build = ctx.path(Label("@rules_haskell//haskell:ghc.BUILD.tpl"))
-
+    paths = resolve_labels(ctx, [
+        "@rules_haskell//haskell:ghc.BUILD.tpl",
+        "@rules_haskell//haskell:private/pkgdb_to_bzl.py",
+    ])
     version = ctx.attr.version
     target = ctx.attr.target
     os, _, arch = target.partition("_")
-    python_bin = _find_python(ctx)
 
     if GHC_BINDIST[version].get(target) == None:
         fail("Operating system {0} does not have a bindist for GHC version {1}".format(ctx.os.name, ctx.attr.version))
@@ -295,6 +304,13 @@ def _ghc_bindist_impl(ctx):
         stripPrefix = "ghc-" + version,
     )
 
+    if os == "windows":
+        # These libraries cause linking errors on Windows when linking
+        # pthreads, due to libwinpthread-1.dll not being loaded.
+        execute_or_fail_loudly(ctx, ["rm", "mingw/lib/gcc/x86_64-w64-mingw32/7.2.0/libstdc++.dll.a"])
+        execute_or_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libpthread.dll.a"])
+        execute_or_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libwinpthread.dll.a"])
+
     # We apply some patches, if needed.
     patch(ctx)
 
@@ -332,73 +348,33 @@ DISTDIR="$( dirname "$(resolved="$0"; cd "$(dirname "$resolved")"; while tmp="$(
         ))
         execute_or_fail_loudly(ctx, ["./patch_bins"])
 
-    # The default locale is OS specific.
-    if ctx.attr.locale:
-        locale = ctx.attr.locale
-    else:
-        locale = "en_US.UTF-8" if os == "darwin" else "C.UTF-8"
-
-    # Generate BUILD file entries describing each prebuilt package.
-    # Cannot use //haskell:pkgdb_to_bzl because that's a generated
-    # target. ctx.path() only works on source files.
-    pkgdb_to_bzl = ctx.path(Label("@rules_haskell//haskell:private/pkgdb_to_bzl.py"))
-    result = ctx.execute([
-        python_bin,
-        pkgdb_to_bzl,
-        ctx.attr.name,
-        "lib",
-    ])
-    if result.return_code:
-        fail("Error executing pkgdb_to_bzl.py: {stderr}".format(stderr = result.stderr))
-    toolchain_libraries = result.stdout
-    toolchain = """
-{toolchain_libraries}
-
-haskell_toolchain(
-    name = "toolchain-impl",
-    tools = [":bin"],
-    libraries = toolchain_libraries,
-    version = "{version}",
-    static_runtime = {static_runtime},
-    fully_static_link = {fully_static_link},
-    compiler_flags = {compiler_flags},
-    haddock_flags = {haddock_flags},
-    repl_ghci_args = {repl_ghci_args},
-    cabalopts = {cabalopts},
-    visibility = ["//visibility:public"],
-    locale = "{locale}",
-)
-    """.format(
-        toolchain_libraries = toolchain_libraries,
-        version = ctx.attr.version,
+    toolchain_libraries = pkgdb_to_bzl(ctx, paths, "lib")
+    locale = ctx.attr.locale or "en_US.UTF-8" if os == "darwin" else "C.UTF-8"
+    toolchain = define_rule(
+        "haskell_toolchain",
+        name = "toolchain-impl",
+        tools = [":bin"],
+        libraries = "toolchain_libraries",
+        # See Note [GHC toolchain files]
+        files = [":lib/settings"],
+        version = repr(ctx.attr.version),
         static_runtime = os == "windows",
-
-        # At present we don't support fully-statically-linked binaries with GHC
-        # bindists.
-        fully_static_link = False,
+        fully_static_link = False,  # XXX not yet supported for bindists.
         compiler_flags = ctx.attr.compiler_flags,
         haddock_flags = ctx.attr.haddock_flags,
         repl_ghci_args = ctx.attr.repl_ghci_args,
         cabalopts = ctx.attr.cabalopts,
-        locale = locale,
+        locale = repr(locale),
     )
-
-    if os == "windows":
-        # These libraries cause linking errors on Windows when linking
-        # pthreads, due to libwinpthread-1.dll not being loaded.
-        execute_or_fail_loudly(ctx, ["rm", "mingw/lib/gcc/x86_64-w64-mingw32/7.2.0/libstdc++.dll.a"])
-        execute_or_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libpthread.dll.a"])
-        execute_or_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libwinpthread.dll.a"])
-
     ctx.template(
         "BUILD",
-        ghc_build,
+        paths["@rules_haskell//haskell:ghc.BUILD.tpl"],
         substitutions = {
+            "%{toolchain_libraries}": toolchain_libraries,
             "%{toolchain}": toolchain,
         },
         executable = False,
     )
-    ctx.file("WORKSPACE")
 
 _ghc_bindist = repository_rule(
     _ghc_bindist_impl,
@@ -590,18 +566,9 @@ def haskell_register_ghc_bindists(
     if local_python_repo_name not in native.existing_rules():
         _configure_python3_toolchain(name = local_python_repo_name)
 
-def _find_python(repository_ctx):
-    python = repository_ctx.which("python3")
-    if not python:
-        python = repository_ctx.which("python")
-        result = repository_ctx.execute([python, "--version"])
-        if not result.stdout.startswith("Python 3"):
-            fail("rules_haskell requires Python >= 3.3.")
-    return python
-
 def _configure_python3_toolchain_impl(repository_ctx):
     cpu = get_cpu_value(repository_ctx)
-    python3_path = _find_python(repository_ctx)
+    python3_path = find_python(repository_ctx)
     repository_ctx.file("BUILD.bazel", executable = False, content = """
 load(
     "@bazel_tools//tools/python:toolchain.bzl",
@@ -683,3 +650,26 @@ def _configure_python3_toolchain(name):
     """
     _config_python3_toolchain(name = name)
     native.register_toolchains("@{}//:toolchain".format(name))
+
+# Note [GHC toolchain files]
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# The GHC distribution includes various files that may be required during
+# compilation, or may be referenced by template Haskell code. These files
+# need to be tracked by Bazel and declared as inputs to the relevant actions
+# to ensure that they are present in the build sandbox.
+#
+# Surprisingly, builds succeed with little to none of such files declared as
+# inputs. In case of the nixpkgs toolchain this is not surprising, as the
+# files will all be present in the Nix store. However, files of the bindist
+# toolchain are fully tracked by Bazel and one would expect errors due to
+# missing files if they are not declared.
+#
+# The first instance of such an error occurred with the GHC bindist on
+# BazelCI [1] and shortly after on the GitHub actions CI pipeline. However,
+# only the `lib/settings` file was reported missing.
+#
+# Do extend the `files` toolchain attributes with further files if such
+# errors occurr in the future.
+#
+# [1]: https://github.com/tweag/rules_haskell/issues/1470

haskell/nixpkgs.bzl

@@ -5,14 +5,25 @@ load(
     "nixpkgs_package",
     "nixpkgs_sh_posix_configure",
 )
+load(
+    ":private/pkgdb_to_bzl.bzl",
+    "pkgdb_to_bzl",
+)
+load(
+    ":private/workspace_utils.bzl",
+    "define_rule",
+    "resolve_labels",
+)
 
 def _ghc_nixpkgs_haskell_toolchain_impl(repository_ctx):
+    paths = resolve_labels(repository_ctx, [
+        "@rules_haskell//haskell:private/pkgdb_to_bzl.py",
+    ])
     compiler_flags_select = "select({})".format(
         repository_ctx.attr.compiler_flags_select or {
             "//conditions:default": [],
         },
     )
-    locale_archive = repository_ctx.attr.locale_archive
     nixpkgs_ghc_path = repository_ctx.path(repository_ctx.attr._nixpkgs_ghc).dirname.dirname
 
     # Symlink content of ghc external repo. In effect, this repo has
@@ -22,8 +33,6 @@ def _ghc_nixpkgs_haskell_toolchain_impl(repository_ctx):
         basename = target.rpartition("/")[-1]
         repository_ctx.symlink(target, basename)
 
-    # Generate BUILD file entries describing each prebuilt package.
-    pkgdb_to_bzl = repository_ctx.path(Label("@rules_haskell//haskell:private/pkgdb_to_bzl.py"))
     ghc_name = "ghc-{}".format(repository_ctx.attr.version)
     result = repository_ctx.execute(["ls", "lib"])
     if result.return_code or not ghc_name in result.stdout.splitlines():
@@ -35,23 +44,28 @@ Available versions:
 {actual}
 """.format(wanted = ghc_name, actual = result.stdout),
         )
-    result = repository_ctx.execute([
-        pkgdb_to_bzl,
-        repository_ctx.attr.name,
-        "lib/{}".format(ghc_name),
-    ])
-    if result.return_code:
-        fail("Error executing pkgdb_to_bzl.py: {stderr}".format(stderr = result.stderr))
-    toolchain_libraries = result.stdout
-
-    # Haddock files on nixpkgs are stored outside of the ghc package
-    # The pkgdb_to_bzl.py program generates bazel labels for theses files
-    # and asks the parent process to generate the associated bazel symlink
-    for line in result.stdout.split("\n"):
-        if line.startswith("#SYMLINK:"):
-            _, path, name = line.split(" ")
-            repository_ctx.symlink(path, name)
-
+    toolchain_libraries = pkgdb_to_bzl(repository_ctx, paths, "lib/{}".format(ghc_name))
+    locale_archive = repository_ctx.attr.locale_archive
+    toolchain = define_rule(
+        "haskell_toolchain",
+        name = "toolchain-impl",
+        libraries = "toolchain_libraries",
+        # See Note [GHC toolchain files] in haskell/ghc_bindist.bzl
+        files = [],
+        tools = ["@rules_haskell_ghc_nixpkgs//:bin"],
+        version = repr(repository_ctx.attr.version),
+        static_runtime = repository_ctx.attr.static_runtime,
+        fully_static_link = repository_ctx.attr.fully_static_link,
+        compiler_flags = "{} + {}".format(
+            repository_ctx.attr.compiler_flags,
+            compiler_flags_select,
+        ),
+        haddock_flags = repository_ctx.attr.haddock_flags,
+        repl_ghci_args = repository_ctx.attr.repl_ghci_args,
+        cabalopts = repository_ctx.attr.cabalopts,
+        locale_archive = repr(locale_archive) if locale_archive else None,
+        locale = repr(repository_ctx.attr.locale),
+    )
     repository_ctx.file(
         "BUILD",
         executable = False,
@@ -71,35 +85,10 @@ filegroup(
 
 {toolchain_libraries}
 
-haskell_toolchain(
-    name = "toolchain-impl",
-    tools = {tools},
-    libraries = toolchain_libraries,
-    version = "{version}",
-    static_runtime = {static_runtime},
-    fully_static_link = {fully_static_link},
-    compiler_flags = {compiler_flags} + {compiler_flags_select},
-    haddock_flags = {haddock_flags},
-    cabalopts = {cabalopts},
-    repl_ghci_args = {repl_ghci_args},
-    # On Darwin we don't need a locale archive. It's a Linux-specific
-    # hack in Nixpkgs.
-    {locale_archive_arg}
-    locale = {locale},
-)
+{toolchain}
         """.format(
             toolchain_libraries = toolchain_libraries,
-            tools = ["@rules_haskell_ghc_nixpkgs//:bin"],
-            version = repository_ctx.attr.version,
-            static_runtime = repository_ctx.attr.static_runtime,
-            fully_static_link = repository_ctx.attr.fully_static_link,
-            compiler_flags = repository_ctx.attr.compiler_flags,
-            compiler_flags_select = compiler_flags_select,
-            haddock_flags = repository_ctx.attr.haddock_flags,
-            repl_ghci_args = repository_ctx.attr.repl_ghci_args,
-            cabalopts = repository_ctx.attr.cabalopts,
-            locale_archive_arg = "locale_archive = {},".format(repr(locale_archive)) if locale_archive else "",
-            locale = repr(repository_ctx.attr.locale),
+            toolchain = toolchain,
         ),
     )
 

haskell/private/pkgdb_to_bzl.bzl

@@ -0,0 +1,31 @@
+load(":private/workspace_utils.bzl", "find_python")
+
+def pkgdb_to_bzl(repository_ctx, paths, libdir):
+    """Generate a BUILD file from a package database.
+
+    Also creates symlinks to any resources stored outside of the GHC
+    base directory.
+
+    Args:
+      repository_ctx: repository context.
+      paths: a dictionary of label names to paths.
+      pkgdir: the relative location of GHC's libdir
+    """
+    result = repository_ctx.execute([
+        find_python(repository_ctx),
+        paths["@rules_haskell//haskell:private/pkgdb_to_bzl.py"],
+        repository_ctx.attr.name,
+        libdir,
+    ])
+    if result.return_code:
+        fail("Error executing pkgdb_to_bzl.py: {stderr}".format(stderr = result.stderr))
+
+    # Haddock files on nixpkgs are stored outside of the ghc package
+    # The pkgdb_to_bzl.py program generates bazel labels for theses files
+    # and asks the parent process to generate the associated bazel symlink
+    for line in result.stdout.split("\n"):
+        if line.startswith("#SYMLINK:"):
+            _, path, name = line.split(" ")
+            repository_ctx.symlink(path, name)
+
+    return result.stdout