From 7f3ab20704c5ffe24292613c7d2febdbc7be1b39 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Fri, 5 May 2023 10:01:42 -0700 Subject: [PATCH 1/5] feature: nixpkgs_flake_package --- README.md | 196 ++++++++++++++++++++++++++++++++++++++ core/nixpkgs.bzl | 225 +++++++++++++++++++++++++++++++++++++++++++- docs/BUILD.bazel | 1 + nixpkgs/nixpkgs.bzl | 3 + 4 files changed, 424 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cb5b75620..b06d3b3cc 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe * [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) +* [nixpkgs_flake_package](#nixpkgs_flake_package) * [nixpkgs_cc_configure](#nixpkgs_cc_configure) * [nixpkgs_cc_configure_deprecated](#nixpkgs_cc_configure_deprecated) * [nixpkgs_java_configure](#nixpkgs_java_configure) @@ -517,6 +518,201 @@ Options to forward to the nix command. + + +### nixpkgs_flake_package + +
+nixpkgs_flake_package(name, nix_flake_file, nix_flake_lock_file, nix_flake_file_deps, package,
+                      build_file, build_file_content, nixopts, quiet, fail_not_supported, kwargs)
+
+ +Make the content of a local Nix Flake package available in the Bazel workspace. + +#### Parameters + + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
name + +required. + +

+ +A unique name for this repository. + +

+
nix_flake_file + +required. + +

+ +Label to `flake.nix` that will be evaluated. + +

+
nix_flake_lock_file + +required. + +

+ +Label to `flake.lock` that corresponds to `nix_flake_file`. + +

+
nix_flake_file_deps + +optional. +default is [] + +

+ +Additional dependencies of `nix_flake_file` if any. + +

+
package + +optional. +default is None + +

+ +Nix Flake package to make available. The default package will be used if not specified. + +

+
build_file + +optional. +default is None + +

+ +The file to use as the BUILD file for this repository. + +Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. + +For common use cases we provide filegroups that expose certain files as targets: + +

+
:bin
+
Everything in the bin/ directory.
+
:lib
+
All .so, .dylib and .a files that can be found in subdirectories of lib/.
+
:include
+
All .h, .hh, .hpp and .hxx files that can be found in subdirectories of include/.
+
+ +If you need different files from the nix package, you can reference them like this: +``` +package(default_visibility = [ "//visibility:public" ]) +filegroup( + name = "our-docs", + srcs = glob(["share/doc/ourpackage/**/*"]), +) +``` +See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). + +

+
build_file_content + +optional. +default is "" + +

+ +Like `build_file`, but a string of the contents instead of a file name. + +

+
nixopts + +optional. +default is [] + +

+ +Extra flags to pass when calling Nix. + +Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file referenced by `LABEL` relative to the workspace root. + +Note, labels to external workspaces will resolve to paths that contain `~` characters if the Bazel flag `--enable_bzlmod` is true. Nix does not support `~` characters in path literals at the time of writing, see [#7742](https://github.com/NixOS/nix/issues/7742). Meaning, the result of location expansion may not form a valid Nix path literal. Use `./$${"$(location @for//:example)"}` to work around this limitation if you need to pass a path argument via `--arg`, or pass the resulting path as a string value using `--argstr` and combine it with an additional `--arg workspace_root ./.` argument using `workspace_root + ("/" + path_str)`. + +

+
quiet + +optional. +default is False + +

+ +Whether to hide the output of the Nix command. + +

+
fail_not_supported + +optional. +default is True + +

+ +If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. + +

+
kwargs + +optional. + +
+ + ### nixpkgs_git_repository diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl index 56ab93e93..983dbb652 100644 --- a/core/nixpkgs.bzl +++ b/core/nixpkgs.bzl @@ -1,4 +1,4 @@ -""" +""" # Nixpkgs rules for Bazel @@ -29,6 +29,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe * [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) +* [nixpkgs_flake_package](#nixpkgs_flake_package) """ load( @@ -453,6 +454,7 @@ def _nixpkgs_package_impl(repository_ctx): else: # No user supplied build file, we may create the default one. create_build_file_if_needed = True + # Workaround to bazelbuild/bazel#4533 repository_ctx.path("BUILD") @@ -668,3 +670,224 @@ def nixpkgs_package( kwargs["repositories"] = inversed_repositories _nixpkgs_package(**kwargs) + +def _nixpkgs_flake_package_impl(repository_ctx): + # Workaround to bazelbuild/bazel#4533 -- to prevent this rule being restarted after running cp, + # resolve all dependencies of this rule before running cp + # + # Remove the following repository_ctx.path() once bazelbuild/bazel#4533 is resolved. + if repository_ctx.attr.build_file: + repository_ctx.path(repository_ctx.attr.build_file) + + repository_ctx.path(repository_ctx.attr.nix_flake_file) + repository_ctx.path(external_repository_root(repository_ctx.attr.nix_flake_file)) + repository_ctx.path(repository_ctx.attr.nix_flake_lock_file) + repository_ctx.path(external_repository_root(repository_ctx.attr.nix_flake_lock_file)) + + for dep in repository_ctx.attr.nix_flake_file_deps: + repository_ctx.path(dep) + repository_ctx.path(external_repository_root(dep)) + + # Is nix supported on this platform? + not_supported = not is_supported_platform(repository_ctx) + + # Should we fail if Nix is not supported? + fail_not_supported = repository_ctx.attr.fail_not_supported + + if not_supported and fail_not_supported: + fail("Platform is not supported: `nix` not found in PATH. See attribute `fail_not_supported` if you don't want to use Nix.") + elif not_supported: + return + + # If true, a BUILD file will be created from a template if it does not + # exist. + # However this will happen AFTER the nix-build command. + create_build_file_if_needed = False + if repository_ctx.attr.build_file and repository_ctx.attr.build_file_content: + fail("Specify one of 'build_file' or 'build_file_content', but not both.") + elif repository_ctx.attr.build_file: + repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") + elif repository_ctx.attr.build_file_content: + repository_ctx.file("BUILD", content = repository_ctx.attr.build_file_content) + else: + # No user supplied build file, we may create the default one. + create_build_file_if_needed = True + + # Workaround to bazelbuild/bazel#4533 + repository_ctx.path("BUILD") + + nix_flake_file_deps = {} + for dep_lbl, dep_str in repository_ctx.attr.nix_flake_file_deps.items(): + nix_flake_file_deps[dep_str] = cp(repository_ctx, dep_lbl) + + nix_build_target = str(repository_ctx.path(repository_ctx.attr.nix_flake_file).dirname) + if repository_ctx.attr.package: + nix_build_target += "#" + repository_ctx.attr.package + + expr_args = [nix_build_target] + + # `nix build` doesn't print the output path by default. + expr_args.extend(["--print-out-paths"]) + + expr_args.extend([ + # Creating an out link prevents nix from garbage collecting the store path. + # nixpkgs uses `nix-support/` for such house-keeping files, so we mirror them + # and use `bazel-support/`, under the assumption that no nix package has + # a file named `bazel-support` in its root. + # A `bazel clean` deletes the symlink and thus nix is free to garbage collect + # the store path. + "--out-link", + "bazel-support/nix-out-link", + ]) + + expr_args.extend([ + expand_location( + repository_ctx = repository_ctx, + string = opt, + labels = nix_flake_file_deps, + attr = "nixopts", + ) + for opt in repository_ctx.attr.nixopts + ]) + + nix_path = executable_path( + repository_ctx, + "nix", + extra_msg = "See: https://nixos.org/nix/", + ) + nix_build = [nix_path, "build"] + expr_args + + # Large enough integer that Bazel can still parse. We don't have + # access to MAX_INT and 0 is not a valid timeout so this is as good + # as we can do. The value shouldn't be too large to avoid errors on + # macOS, see https://github.com/tweag/rules_nixpkgs/issues/92. + timeout = 8640000 + repository_ctx.report_progress("Building Nix derivation") + exec_result = execute_or_fail( + repository_ctx, + nix_build, + failure_message = "Cannot build Nix Flake '{}'.".format(nix_build_target), + quiet = repository_ctx.attr.quiet, + timeout = timeout, + ) + output_path = exec_result.stdout.splitlines()[-1] + + # ensure that the output is a directory + test_path = repository_ctx.which("test") + execute_or_fail( + repository_ctx, + [test_path, "-d", output_path], + failure_message = "Nix Flake '{}' outputs a single file which is not supported by rules_nixpkgs. Please only use directories.".format( + nix_build_target, + ), + ) + + # Build a forest of symlinks (like new_local_package() does) to the + # Nix store. + for target in find_children(repository_ctx, output_path): + basename = target.rpartition("/")[-1] + repository_ctx.symlink(target, basename) + + # Create a default BUILD file only if it does not exists and is not + # provided by `build_file` or `build_file_content`. + if create_build_file_if_needed: + p = repository_ctx.path("BUILD") + if not p.exists: + repository_ctx.template("BUILD", Label("@rules_nixpkgs_core//:BUILD.bazel.tpl")) + +_nixpkgs_flake_package = repository_rule( + implementation = _nixpkgs_flake_package_impl, + attrs = { + "nix_flake_file": attr.label(mandatory = True, allow_single_file = ["flake.nix"]), + "nix_flake_lock_file": attr.label(mandatory = True, allow_single_file = ["flake.lock"]), + "nix_flake_file_deps": attr.label_keyed_string_dict(), + "package": attr.string(doc = "Defaults to `default`"), + "build_file": attr.label(), + "build_file_content": attr.string(), + "nixopts": attr.string_list(), + "quiet": attr.bool(), + "fail_not_supported": attr.bool(default = True, doc = """ + If set to True (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to False calling this rule will succeed but no output will be generated. + """), + }, +) + +def nixpkgs_flake_package( + name, + nix_flake_file, + nix_flake_lock_file, + nix_flake_file_deps = [], + package = None, + build_file = None, + build_file_content = "", + nixopts = [], + quiet = False, + fail_not_supported = True, + **kwargs): + """Make the content of a local Nix Flake package available in the Bazel workspace. + + Args: + name: A unique name for this repository. + nix_flake_file: Label to `flake.nix` that will be evaluated. + nix_flake_lock_file: Label to `flake.lock` that corresponds to `nix_flake_file`. + nix_flake_file_deps: Additional dependencies of `nix_flake_file` if any. + package: Nix Flake package to make available. The default package will be used if not specified. + build_file: The file to use as the BUILD file for this repository. + + Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. + + For common use cases we provide filegroups that expose certain files as targets: + +
+
:bin
+
Everything in the bin/ directory.
+
:lib
+
All .so, .dylib and .a files that can be found in subdirectories of lib/.
+
:include
+
All .h, .hh, .hpp and .hxx files that can be found in subdirectories of include/.
+
+ + If you need different files from the nix package, you can reference them like this: + ``` + package(default_visibility = [ "//visibility:public" ]) + filegroup( + name = "our-docs", + srcs = glob(["share/doc/ourpackage/**/*"]), + ) + ``` + See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). + build_file_content: Like `build_file`, but a string of the contents instead of a file name. + nixopts: Extra flags to pass when calling Nix. + + Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file referenced by `LABEL` relative to the workspace root. + + Note, labels to external workspaces will resolve to paths that contain `~` characters if the Bazel flag `--enable_bzlmod` is true. Nix does not support `~` characters in path literals at the time of writing, see [#7742](https://github.com/NixOS/nix/issues/7742). Meaning, the result of location expansion may not form a valid Nix path literal. Use `./$${"$(location @for//:example)"}` to work around this limitation if you need to pass a path argument via `--arg`, or pass the resulting path as a string value using `--argstr` and combine it with an additional `--arg workspace_root ./.` argument using `workspace_root + ("/" + path_str)`. + quiet: Whether to hide the output of the Nix command. + fail_not_supported: If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. + """ + if kwargs.pop("_bzlmod", None): + # The workaround to map canonicalized labels to the user provided + # string representation to enable location expansion does not work when + # nixpkgs_package is invoked from a module extension, because module + # extension tags cannot be wrapped in macros. + # Until we find a solution to this issue, we provide the canonicalized + # label as a string. Location expansion will have to be performed on + # canonicalized labels until a better solution is found. + # TODO[AH] Support proper location expansion in module extension. + nix_flake_file_deps = {dep: str(dep) for dep in nix_flake_file_deps} if nix_flake_file_deps else {} + else: + nix_flake_file_deps = {dep: dep for dep in nix_flake_file_deps} if nix_flake_file_deps else {} + kwargs.update( + name = name, + nix_flake_file = nix_flake_file, + nix_flake_lock_file = nix_flake_lock_file, + nix_flake_file_deps = nix_flake_file_deps, + package = package, + build_file = build_file, + build_file_content = build_file_content, + nixopts = nixopts, + quiet = quiet, + fail_not_supported = fail_not_supported, + ) + + _nixpkgs_flake_package(**kwargs) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 90ba34e32..397762347 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -14,6 +14,7 @@ generate_documentation( "nixpkgs_http_repository", "nixpkgs_local_repository", "nixpkgs_package", + "nixpkgs_flake_package", "nixpkgs_cc_configure", "nixpkgs_cc_configure_deprecated", "nixpkgs_go_configure", diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 693a78a91..9ae977e9e 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -29,6 +29,7 @@ See [examples](/examples/toolchains) for how to use `rules_nixpkgs` with differe * [nixpkgs_http_repository](#nixpkgs_http_repository) * [nixpkgs_local_repository](#nixpkgs_local_repository) * [nixpkgs_package](#nixpkgs_package) +* [nixpkgs_flake_package](#nixpkgs_flake_package) * [nixpkgs_cc_configure](#nixpkgs_cc_configure) * [nixpkgs_cc_configure_deprecated](#nixpkgs_cc_configure_deprecated) * [nixpkgs_java_configure](#nixpkgs_java_configure) @@ -118,6 +119,7 @@ load( ) load( "@rules_nixpkgs_core//:nixpkgs.bzl", + _nixpkgs_flake_package = "nixpkgs_flake_package", _nixpkgs_git_repository = "nixpkgs_git_repository", _nixpkgs_http_repository = "nixpkgs_http_repository", _nixpkgs_local_repository = "nixpkgs_local_repository", @@ -162,6 +164,7 @@ nixpkgs_git_repository = _nixpkgs_git_repository nixpkgs_http_repository = _nixpkgs_http_repository nixpkgs_local_repository = _nixpkgs_local_repository nixpkgs_package = _nixpkgs_package +nixpkgs_flake_package = _nixpkgs_flake_package nixpkgs_python_configure = _nixpkgs_python_configure nixpkgs_python_repository = _nixpkgs_python_repository nixpkgs_java_configure = _nixpkgs_java_configure From 7c8bd9415f5e54cce9a5007bdbe7cc3bdb0e3a99 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Mon, 8 May 2023 09:48:42 -0700 Subject: [PATCH 2/5] Delete `flake.lock` symlink --- testing/core/flake.lock | 1 - 1 file changed, 1 deletion(-) delete mode 120000 testing/core/flake.lock diff --git a/testing/core/flake.lock b/testing/core/flake.lock deleted file mode 120000 index 23b0a9b50..000000000 --- a/testing/core/flake.lock +++ /dev/null @@ -1 +0,0 @@ -../../flake.lock \ No newline at end of file From 645e6a826bb9123ac338bd56615dad23dff1bee8 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Tue, 9 May 2023 17:25:39 -0700 Subject: [PATCH 3/5] Refactor + Tests --- core/nixpkgs.bzl | 213 +++++++------------- testing/core/flake.lock | 78 +++++++ testing/core/flake.nix | 26 +++ testing/core/tests/BUILD.bazel | 2 + testing/core/tests/nixpkgs_repositories.bzl | 15 ++ 5 files changed, 189 insertions(+), 145 deletions(-) create mode 100644 testing/core/flake.lock create mode 100644 testing/core/flake.nix diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl index 983dbb652..d3f201eac 100644 --- a/core/nixpkgs.bzl +++ b/core/nixpkgs.bzl @@ -378,6 +378,66 @@ def nixpkgs_local_repository( **kwargs ) +def _nixpkgs_build_file_content(repository_ctx): + # Workaround to bazelbuild/bazel#4533 + repository_ctx.path("BUILD") + repository_ctx.path("BUILD.bazel") + + if repository_ctx.attr.build_file: + repository_ctx.path(repository_ctx.attr.build_file) + + if repository_ctx.attr.build_file and repository_ctx.attr.build_file_content: + fail("Specify one of 'build_file' or 'build_file_content', but not both.") + + if repository_ctx.attr.build_file: + return repository_ctx.read(repository_ctx.attr.build_file) + elif repository_ctx.attr.build_file_content: + return repository_ctx.attr.build_file_content + else: + return None + +def _nixpkgs_build_and_symlink(repository_ctx, nix_build, build_file_content): + # Large enough integer that Bazel can still parse. We don't have + # access to MAX_INT and 0 is not a valid timeout so this is as good + # as we can do. The value shouldn't be too large to avoid errors on + # macOS, see https://github.com/tweag/rules_nixpkgs/issues/92. + timeout = 8640000 + repository_ctx.report_progress("Building Nix derivation") + exec_result = execute_or_fail( + repository_ctx, + nix_build, + failure_message = "Cannot build Nix derivation for package '@{}'.".format(repository_ctx.name), + quiet = repository_ctx.attr.quiet, + timeout = timeout, + ) + output_path = exec_result.stdout.splitlines()[-1] + + # ensure that the output is a directory + test_path = repository_ctx.which("test") + execute_or_fail( + repository_ctx, + [test_path, "-d", output_path], + failure_message = "Package '@{}' outputs a single file which is not supported by rules_nixpkgs. Please only use directories.".format( + repository_ctx.name, + ), + ) + + # Build a forest of symlinks (like new_local_package() does) to the + # Nix store. + for target in find_children(repository_ctx, output_path): + basename = target.rpartition("/")[-1] + repository_ctx.symlink(target, basename) + + # Create a default BUILD file only if it does not exists and is not + # provided by `build_file` or `build_file_content`. + if not repository_ctx.path("BUILD").exists and not repository_ctx.path("BUILD.bazel").exists: + if build_file_content: + repository_ctx.file("BUILD", content = build_file_content) + else: + repository_ctx.template("BUILD", Label("@rules_nixpkgs_core//:BUILD.bazel.tpl")) + elif build_file_content: + fail("One of 'build_file' or 'build_file_content' was specified but Nix derivation already contains 'BUILD' or 'BUILD.bazel'.") + def _nixpkgs_package_impl(repository_ctx): repository = repository_ctx.attr.repository repositories = repository_ctx.attr.repositories @@ -419,8 +479,7 @@ def _nixpkgs_package_impl(repository_ctx): # resolve all dependencies of this rule before running cp # # Remove the following repository_ctx.path() once bazelbuild/bazel#4533 is resolved. - if repository_ctx.attr.build_file: - repository_ctx.path(repository_ctx.attr.build_file) + build_file_content = _nixpkgs_build_file_content(repository_ctx) if repository_ctx.attr.nix_file: repository_ctx.path(repository_ctx.attr.nix_file) @@ -441,23 +500,6 @@ def _nixpkgs_package_impl(repository_ctx): elif not_supported: return - # If true, a BUILD file will be created from a template if it does not - # exist. - # However this will happen AFTER the nix-build command. - create_build_file_if_needed = False - if repository_ctx.attr.build_file and repository_ctx.attr.build_file_content: - fail("Specify one of 'build_file' or 'build_file_content', but not both.") - elif repository_ctx.attr.build_file: - repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") - elif repository_ctx.attr.build_file_content: - repository_ctx.file("BUILD", content = repository_ctx.attr.build_file_content) - else: - # No user supplied build file, we may create the default one. - create_build_file_if_needed = True - - # Workaround to bazelbuild/bazel#4533 - repository_ctx.path("BUILD") - if repository_ctx.attr.nix_file and repository_ctx.attr.nix_file_content: fail("Specify one of 'nix_file' or 'nix_file_content', but not both.") elif repository_ctx.attr.nix_file: @@ -502,45 +544,7 @@ def _nixpkgs_package_impl(repository_ctx): ) nix_build = [nix_build_path] + expr_args - # Large enough integer that Bazel can still parse. We don't have - # access to MAX_INT and 0 is not a valid timeout so this is as good - # as we can do. The value shouldn't be too large to avoid errors on - # macOS, see https://github.com/tweag/rules_nixpkgs/issues/92. - timeout = 8640000 - repository_ctx.report_progress("Building Nix derivation") - exec_result = execute_or_fail( - repository_ctx, - nix_build, - failure_message = "Cannot build Nix attribute '{}'.".format( - repository_ctx.attr.attribute_path, - ), - quiet = repository_ctx.attr.quiet, - timeout = timeout, - ) - output_path = exec_result.stdout.splitlines()[-1] - - # ensure that the output is a directory - test_path = repository_ctx.which("test") - execute_or_fail( - repository_ctx, - [test_path, "-d", output_path], - failure_message = "nixpkgs_package '@{}' outputs a single file which is not supported by rules_nixpkgs. Please only use directories.".format( - repository_ctx.name, - ), - ) - - # Build a forest of symlinks (like new_local_package() does) to the - # Nix store. - for target in find_children(repository_ctx, output_path): - basename = target.rpartition("/")[-1] - repository_ctx.symlink(target, basename) - - # Create a default BUILD file only if it does not exists and is not - # provided by `build_file` or `build_file_content`. - if create_build_file_if_needed: - p = repository_ctx.path("BUILD") - if not p.exists: - repository_ctx.template("BUILD", Label("@rules_nixpkgs_core//:BUILD.bazel.tpl")) + _nixpkgs_build_and_symlink(repository_ctx, nix_build, build_file_content) _nixpkgs_package = repository_rule( implementation = _nixpkgs_package_impl, @@ -601,7 +605,7 @@ def nixpkgs_package( Specify one of `repository` or `repositories`. build_file: The file to use as the BUILD file for this repository. - Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. + Its contents are copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. For common use cases we provide filegroups that expose certain files as targets: @@ -676,8 +680,7 @@ def _nixpkgs_flake_package_impl(repository_ctx): # resolve all dependencies of this rule before running cp # # Remove the following repository_ctx.path() once bazelbuild/bazel#4533 is resolved. - if repository_ctx.attr.build_file: - repository_ctx.path(repository_ctx.attr.build_file) + build_file_content = _nixpkgs_build_file_content(repository_ctx) repository_ctx.path(repository_ctx.attr.nix_flake_file) repository_ctx.path(external_repository_root(repository_ctx.attr.nix_flake_file)) @@ -699,23 +702,6 @@ def _nixpkgs_flake_package_impl(repository_ctx): elif not_supported: return - # If true, a BUILD file will be created from a template if it does not - # exist. - # However this will happen AFTER the nix-build command. - create_build_file_if_needed = False - if repository_ctx.attr.build_file and repository_ctx.attr.build_file_content: - fail("Specify one of 'build_file' or 'build_file_content', but not both.") - elif repository_ctx.attr.build_file: - repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") - elif repository_ctx.attr.build_file_content: - repository_ctx.file("BUILD", content = repository_ctx.attr.build_file_content) - else: - # No user supplied build file, we may create the default one. - create_build_file_if_needed = True - - # Workaround to bazelbuild/bazel#4533 - repository_ctx.path("BUILD") - nix_flake_file_deps = {} for dep_lbl, dep_str in repository_ctx.attr.nix_flake_file_deps.items(): nix_flake_file_deps[dep_str] = cp(repository_ctx, dep_lbl) @@ -757,43 +743,7 @@ def _nixpkgs_flake_package_impl(repository_ctx): ) nix_build = [nix_path, "build"] + expr_args - # Large enough integer that Bazel can still parse. We don't have - # access to MAX_INT and 0 is not a valid timeout so this is as good - # as we can do. The value shouldn't be too large to avoid errors on - # macOS, see https://github.com/tweag/rules_nixpkgs/issues/92. - timeout = 8640000 - repository_ctx.report_progress("Building Nix derivation") - exec_result = execute_or_fail( - repository_ctx, - nix_build, - failure_message = "Cannot build Nix Flake '{}'.".format(nix_build_target), - quiet = repository_ctx.attr.quiet, - timeout = timeout, - ) - output_path = exec_result.stdout.splitlines()[-1] - - # ensure that the output is a directory - test_path = repository_ctx.which("test") - execute_or_fail( - repository_ctx, - [test_path, "-d", output_path], - failure_message = "Nix Flake '{}' outputs a single file which is not supported by rules_nixpkgs. Please only use directories.".format( - nix_build_target, - ), - ) - - # Build a forest of symlinks (like new_local_package() does) to the - # Nix store. - for target in find_children(repository_ctx, output_path): - basename = target.rpartition("/")[-1] - repository_ctx.symlink(target, basename) - - # Create a default BUILD file only if it does not exists and is not - # provided by `build_file` or `build_file_content`. - if create_build_file_if_needed: - p = repository_ctx.path("BUILD") - if not p.exists: - repository_ctx.template("BUILD", Label("@rules_nixpkgs_core//:BUILD.bazel.tpl")) + _nixpkgs_build_and_symlink(repository_ctx, nix_build, build_file_content) _nixpkgs_flake_package = repository_rule( implementation = _nixpkgs_flake_package_impl, @@ -832,36 +782,9 @@ def nixpkgs_flake_package( nix_flake_lock_file: Label to `flake.lock` that corresponds to `nix_flake_file`. nix_flake_file_deps: Additional dependencies of `nix_flake_file` if any. package: Nix Flake package to make available. The default package will be used if not specified. - build_file: The file to use as the BUILD file for this repository. - - Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. - - For common use cases we provide filegroups that expose certain files as targets: - -
-
:bin
-
Everything in the bin/ directory.
-
:lib
-
All .so, .dylib and .a files that can be found in subdirectories of lib/.
-
:include
-
All .h, .hh, .hpp and .hxx files that can be found in subdirectories of include/.
-
- - If you need different files from the nix package, you can reference them like this: - ``` - package(default_visibility = [ "//visibility:public" ]) - filegroup( - name = "our-docs", - srcs = glob(["share/doc/ourpackage/**/*"]), - ) - ``` - See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). - build_file_content: Like `build_file`, but a string of the contents instead of a file name. - nixopts: Extra flags to pass when calling Nix. - - Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file referenced by `LABEL` relative to the workspace root. - - Note, labels to external workspaces will resolve to paths that contain `~` characters if the Bazel flag `--enable_bzlmod` is true. Nix does not support `~` characters in path literals at the time of writing, see [#7742](https://github.com/NixOS/nix/issues/7742). Meaning, the result of location expansion may not form a valid Nix path literal. Use `./$${"$(location @for//:example)"}` to work around this limitation if you need to pass a path argument via `--arg`, or pass the resulting path as a string value using `--argstr` and combine it with an additional `--arg workspace_root ./.` argument using `workspace_root + ("/" + path_str)`. + build_file: The file to use as the BUILD file for this repository. See [`nixpkgs_package`](#nixpkgs_package) for more information. + build_file_content: Like `build_file`, but a string of the contents instead of a file name. See [`nixpkgs_package`](#nixpkgs_package) for more information. + nixopts: Extra flags to pass when calling Nix. See [`nixpkgs_package`](#nixpkgs_package) for more information. quiet: Whether to hide the output of the Nix command. fail_not_supported: If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. """ diff --git a/testing/core/flake.lock b/testing/core/flake.lock new file mode 100644 index 000000000..abae48530 --- /dev/null +++ b/testing/core/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1683478192, + "narHash": "sha256-7f7RR71w0jRABDgBwjq3vE1yY3nrVJyXk8hDzu5kl1E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c568239bcc990050b7aedadb7387832440ad8fb1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/testing/core/flake.nix b/testing/core/flake.nix new file mode 100644 index 000000000..305b989c1 --- /dev/null +++ b/testing/core/flake.nix @@ -0,0 +1,26 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11"; + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; + flake-utils.url = "github:numtide/flake-utils"; + }; + outputs = { nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + in + { + packages.hello = pkgs.hello; + + # Custom `BUILD.bazel` that resolves `hello` in its non-default location + packages.hello-with-build-file = with pkgs; runCommandLocal "hello-with-build-file" { } '' + mkdir --parents $out + ln -s ${hello}/bin $out/bin-hidden + ln -s ${hello}/share $out/share-hidden + echo 'filegroup(name = "bin", srcs = ["bin-hidden/hello"], visibility = ["//visibility:public"])' > $out/BUILD.bazel + ''; + }); +} diff --git a/testing/core/tests/BUILD.bazel b/testing/core/tests/BUILD.bazel index 6437addc6..6715dad4f 100644 --- a/testing/core/tests/BUILD.bazel +++ b/testing/core/tests/BUILD.bazel @@ -31,6 +31,8 @@ expand_location_unit_test_suite() "expr-attribute-test", "relative-imports", "extra-args-test", + "flake-hello", + "flake-hello-with-build-file", ] ] + [ # These tests use the nix package generated by ./output.nix diff --git a/testing/core/tests/nixpkgs_repositories.bzl b/testing/core/tests/nixpkgs_repositories.bzl index 42d2cf256..039af4365 100644 --- a/testing/core/tests/nixpkgs_repositories.bzl +++ b/testing/core/tests/nixpkgs_repositories.bzl @@ -2,6 +2,7 @@ load("@nixpkgs_repositories//:defs.bzl", "nix_repo") load("@nixpkgs_packages//:defs.bzl", "nix_pkg") load( "@rules_nixpkgs_core//:nixpkgs.bzl", + "nixpkgs_flake_package", "nixpkgs_git_repository", "nixpkgs_http_repository", "nixpkgs_local_repository", @@ -210,3 +211,17 @@ filegroup( ], repository = nixpkgs, ) + + nixpkgs_flake_package( + name = "flake-hello", + nix_flake_file = "//:flake.nix", + nix_flake_lock_file = "//:flake.lock", + package = "hello", + ) + + nixpkgs_flake_package( + name = "flake-hello-with-build-file", + nix_flake_file = "//:flake.nix", + nix_flake_lock_file = "//:flake.lock", + package = "hello-with-build-file", + ) From 23d9c921d78a12b33325e3ff2cb0abef0bf648e9 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Thu, 11 May 2023 09:40:13 -0700 Subject: [PATCH 4/5] Documentation fix --- README.md | 35 ++++------------------------------- core/nixpkgs.bzl | 6 +++--- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b06d3b3cc..ebb68381f 100644 --- a/README.md +++ b/README.md @@ -613,30 +613,7 @@ default is None

-The file to use as the BUILD file for this repository. - -Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. - -For common use cases we provide filegroups that expose certain files as targets: - -

-
:bin
-
Everything in the bin/ directory.
-
:lib
-
All .so, .dylib and .a files that can be found in subdirectories of lib/.
-
:include
-
All .h, .hh, .hpp and .hxx files that can be found in subdirectories of include/.
-
- -If you need different files from the nix package, you can reference them like this: -``` -package(default_visibility = [ "//visibility:public" ]) -filegroup( - name = "our-docs", - srcs = glob(["share/doc/ourpackage/**/*"]), -) -``` -See the bazel documentation of [`filegroup`](https://docs.bazel.build/versions/master/be/general.html#filegroup) and [`glob`](https://docs.bazel.build/versions/master/be/functions.html#glob). +The file to use as the BUILD file for this repository. See [`nixpkgs_package`](#nixpkgs_package-build_file) for more information.

@@ -650,7 +627,7 @@ default is ""

-Like `build_file`, but a string of the contents instead of a file name. +Like `build_file`, but a string of the contents instead of a file name. See [`nixpkgs_package`](#nixpkgs_package-build_file_content) for more information.

@@ -664,11 +641,7 @@ default is []

-Extra flags to pass when calling Nix. - -Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file referenced by `LABEL` relative to the workspace root. - -Note, labels to external workspaces will resolve to paths that contain `~` characters if the Bazel flag `--enable_bzlmod` is true. Nix does not support `~` characters in path literals at the time of writing, see [#7742](https://github.com/NixOS/nix/issues/7742). Meaning, the result of location expansion may not form a valid Nix path literal. Use `./$${"$(location @for//:example)"}` to work around this limitation if you need to pass a path argument via `--arg`, or pass the resulting path as a string value using `--argstr` and combine it with an additional `--arg workspace_root ./.` argument using `workspace_root + ("/" + path_str)`. +Extra flags to pass when calling Nix. See [`nixpkgs_package`](#nixpkgs_package-nixopts) for more information.

@@ -1946,7 +1919,7 @@ default is None The file to use as the BUILD file for this repository. -Its contents are copied copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. +Its contents are copied into the file `BUILD` in root of the nix output folder. The Label does not need to be named `BUILD`, but can be. For common use cases we provide filegroups that expose certain files as targets: diff --git a/core/nixpkgs.bzl b/core/nixpkgs.bzl index d3f201eac..1e28fa5af 100644 --- a/core/nixpkgs.bzl +++ b/core/nixpkgs.bzl @@ -782,9 +782,9 @@ def nixpkgs_flake_package( nix_flake_lock_file: Label to `flake.lock` that corresponds to `nix_flake_file`. nix_flake_file_deps: Additional dependencies of `nix_flake_file` if any. package: Nix Flake package to make available. The default package will be used if not specified. - build_file: The file to use as the BUILD file for this repository. See [`nixpkgs_package`](#nixpkgs_package) for more information. - build_file_content: Like `build_file`, but a string of the contents instead of a file name. See [`nixpkgs_package`](#nixpkgs_package) for more information. - nixopts: Extra flags to pass when calling Nix. See [`nixpkgs_package`](#nixpkgs_package) for more information. + build_file: The file to use as the BUILD file for this repository. See [`nixpkgs_package`](#nixpkgs_package-build_file) for more information. + build_file_content: Like `build_file`, but a string of the contents instead of a file name. See [`nixpkgs_package`](#nixpkgs_package-build_file_content) for more information. + nixopts: Extra flags to pass when calling Nix. See [`nixpkgs_package`](#nixpkgs_package-nixopts) for more information. quiet: Whether to hide the output of the Nix command. fail_not_supported: If set to `True` (default) this rule will fail on platforms which do not support Nix (e.g. Windows). If set to `False` calling this rule will succeed but no output will be generated. """ From aea405f1e28b50fa86a39af37cf1e85038011689 Mon Sep 17 00:00:00 2001 From: James Leitch Date: Sun, 14 May 2023 17:26:40 -0700 Subject: [PATCH 5/5] bzlmod test fix --- testing/core/MODULE.bazel | 2 ++ testing/core/tests/BUILD.bazel | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/testing/core/MODULE.bazel b/testing/core/MODULE.bazel index 40764dc63..ae934bac1 100644 --- a/testing/core/MODULE.bazel +++ b/testing/core/MODULE.bazel @@ -114,3 +114,5 @@ nix_pkg.local_expr( non_module_deps = use_extension("//:non_module_deps.bzl", "non_module_deps") use_repo(non_module_deps, "nixpkgs_location_expansion_test") +use_repo(non_module_deps, "flake-hello") +use_repo(non_module_deps, "flake-hello-with-build-file") diff --git a/testing/core/tests/BUILD.bazel b/testing/core/tests/BUILD.bazel index 6715dad4f..eac05ed1c 100644 --- a/testing/core/tests/BUILD.bazel +++ b/testing/core/tests/BUILD.bazel @@ -31,8 +31,6 @@ expand_location_unit_test_suite() "expr-attribute-test", "relative-imports", "extra-args-test", - "flake-hello", - "flake-hello-with-build-file", ] ] + [ # These tests use the nix package generated by ./output.nix @@ -63,6 +61,21 @@ expand_location_unit_test_suite() ], data = [nix_pkg("rules_nixpkgs_core_testing", "output-filegroup-manual-test", "//:manual-filegroup")], ), +] + [ + # All of these tests use the "hello" binary to see + # whether different invocations of `nixpkgs_flake_package` + # produce a valid bazel repository. + sh_test( + name = "run-{0}".format(test), + timeout = "short", + srcs = ["test_bin.sh"], + args = ["$(location @{0}//:bin)".format(test)], + data = ["@{0}//:bin".format(test)], + ) + for test in [ + "flake-hello", + "flake-hello-with-build-file", + ] ] # Test nixopts location expansion