From 3a8816b69fece0068605c055277fa4a9cab539de Mon Sep 17 00:00:00 2001 From: Pierre-Etienne Meunier Date: Mon, 16 Dec 2024 15:35:57 +0100 Subject: [PATCH] Fix tests --- .github/workflows/tests-nix-macos.yml | 2 +- crate2nix/Cargo.nix | 1287 ++++++++--------- crate2nix/src/lib.rs | 5 +- crate2nix/src/prefetch.rs | 5 +- crate2nix/templates/nix/crate2nix/default.nix | 870 ++++++----- .../bin_with_git_submodule_dep/Cargo.nix | 859 ++++++----- sample_projects/codegen/Cargo.nix | 839 ++++++----- sample_projects/sub_dir_crates/Cargo.nix | 859 ++++++----- 8 files changed, 2576 insertions(+), 2150 deletions(-) diff --git a/.github/workflows/tests-nix-macos.yml b/.github/workflows/tests-nix-macos.yml index d4309390..02b139b7 100644 --- a/.github/workflows/tests-nix-macos.yml +++ b/.github/workflows/tests-nix-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: matrix: platform: - - macos-12 + - macos-13 runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v4 diff --git a/crate2nix/Cargo.nix b/crate2nix/Cargo.nix index edc0ffe0..532fc7c1 100644 --- a/crate2nix/Cargo.nix +++ b/crate2nix/Cargo.nix @@ -1,10 +1,9 @@ - # This file was @generated by crate2nix 0.14.1 with the command: # "generate" "-n" "../nix/nixpkgs.nix" "-f" "./crate2nix/Cargo.toml" "-o" "./crate2nix/Cargo.nix" # See https://github.com/kolloch/crate2nix for more info. { nixpkgs ? ../nix/nixpkgs.nix -, pkgs ? import nixpkgs { config = {}; } +, pkgs ? import nixpkgs { config = { }; } , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? pkgs: pkgs.buildRustCrate @@ -17,14 +16,13 @@ # Elements to add to the `-C target-feature=` argument passed to `rustc` # (separated by `,`, prefixed with `+`). # Used for conditional compilation based on CPU feature detection. -, targetFeatures ? [] +, targetFeatures ? [ ] # Whether to perform release builds: longer compile times, faster binaries. , release ? true # Additional crate2nix configuration if it exists. -, crateConfig - ? if builtins.pathExists ./crate-config.nix - then pkgs.callPackage ./crate-config.nix {} - else {} +, crateConfig ? if builtins.pathExists ./crate-config.nix + then pkgs.callPackage ./crate-config.nix { } + else { } }: rec { @@ -63,10 +61,10 @@ rec { # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { - name = "all-workspace-members"; - paths = - let members = builtins.attrValues workspaceMembers; - in builtins.map (m: m.build) members; + name = "all-workspace-members"; + paths = + let members = builtins.attrValues workspaceMembers; + in builtins.map (m: m.build) members; }; # @@ -477,7 +475,7 @@ rec { requiredFeatures = [ ]; } ]; - src = lib.cleanSourceWith { filter = sourceFilter; src = ./.; }; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./.; }; authors = [ "Peter Kolloch " ]; @@ -996,8 +994,7 @@ rec { target = { target, features }: (target."windows" or false); } ]; - features = { - }; + features = { }; }; "indexmap" = rec { crateName = "indexmap"; @@ -1821,8 +1818,7 @@ rec { features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } ]; - features = { - }; + features = { }; resolvedDefaultFeatures = [ "default" ]; }; "serde_json" = rec { @@ -2008,8 +2004,7 @@ rec { features = [ "full" ]; } ]; - features = { - }; + features = { }; }; "syn 1.0.109" = rec { crateName = "syn"; @@ -2428,8 +2423,7 @@ rec { authors = [ "The UNIC Project Developers" ]; - features = { - }; + features = { }; resolvedDefaultFeatures = [ "default" ]; }; "unic-segment" = rec { @@ -2547,8 +2541,7 @@ rec { "kwantam " "Manish Goregaokar " ]; - features = { - }; + features = { }; }; "unicode-width" = rec { crateName = "unicode-width"; @@ -3143,670 +3136,670 @@ rec { }; # -# crate2nix/default.nix (excerpt start) -# - - /* Target (platform) data for conditional dependencies. - This corresponds roughly to what buildRustCrate is setting. - */ - makeDefaultTarget = platform: { - unix = platform.isUnix; - windows = platform.isWindows; - fuchsia = true; - test = false; - - inherit (platform.rust.platform) - arch - os - vendor; - family = platform.rust.platform.target-family; - env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; - pointer_width = toString platform.parsed.cpu.bits; - debug_assertions = false; - }; + # crate2nix/default.nix (excerpt start) + # - /* Filters common temp files and build files. */ - # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: - let - baseName = builtins.baseNameOf (builtins.toString name); - in - ! ( - # Filter out git - baseName == ".gitignore" - || (type == "directory" && baseName == ".git") - - # Filter out build results - || ( - type == "directory" && ( - baseName == "target" - || baseName == "_site" - || baseName == ".sass-cache" - || baseName == ".jekyll-metadata" - || baseName == "build-artifacts" - ) - ) + /* Target (platform) data for conditional dependencies. + This corresponds roughly to what buildRustCrate is setting. + */ + makeDefaultTarget = platform: { + unix = platform.isUnix; + windows = platform.isWindows; + fuchsia = true; + test = false; + + inherit (platform.rust.platform) + arch + os + vendor; + family = platform.rust.platform.target-family; + env = "gnu"; + endian = + if platform.parsed.cpu.significantByte.name == "littleEndian" + then "little" else "big"; + pointer_width = toString platform.parsed.cpu.bits; + debug_assertions = false; + }; - # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + /* Filters common temp files and build files. */ + # TODO(pkolloch): Substitute with gitignore filter + sourceFilter = name: type: + let + baseName = builtins.baseNameOf (builtins.toString name); + in + ! ( + # Filter out git + baseName == ".gitignore" + || (type == "directory" && baseName == ".git") + + # Filter out build results + || ( + type == "directory" && ( + baseName == "target" + || baseName == "_site" + || baseName == ".sass-cache" + || baseName == ".jekyll-metadata" + || baseName == "build-artifacts" + ) + ) - # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" + # Filter out nix-build result symlinks + || ( + type == "symlink" && lib.hasPrefix "result" baseName ) - ) || lib.hasSuffix ".iml" baseName - - # Filter out nix build files - || baseName == "Cargo.nix" - - # Filter out editor backup / swap files. - || lib.hasSuffix "~" baseName - || builtins.match "^\\.sw[a-z]$$" baseName != null - || builtins.match "^\\..*\\.sw[a-z]$$" baseName != null - || lib.hasSuffix ".tmp" baseName - || lib.hasSuffix ".bak" baseName - || baseName == "tests.nix" - ); - - /* Returns a crate which depends on successful test execution - of crate given as the second argument. - - testCrateFlags: list of flags to pass to the test exectuable - testInputs: list of packages that should be available during test execution - */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - } - ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.stdenvNoCC.mkDerivation { - name = "run-tests-${testCrate.name}"; - - inherit (crate) src; - - inherit testCrateFlags; - - buildInputs = testInputs; - - buildPhase = '' - set -e - export RUST_BACKTRACE=1 - - # build outputs - testRoot=target/debug - mkdir -p $testRoot - - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . - - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - }; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); - - /* A restricted overridable version of builtRustCratesWithFeatures. */ - buildRustCrateWithFeatures = - { packageId - , features ? rootFeatures - , crateOverrides ? defaultCrateOverrides - , buildRustCrateForPkgsFunc ? null - , runTests ? false - , testCrateFlags ? [ ] - , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" - }: - lib.makeOverridable - ( - { features - , crateOverrides - , runTests - , testCrateFlags - , testInputs - , testPreRun - , testPostRun - }: - let - buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc - else + + # Filter out IDE config + || ( + type == "directory" && ( + baseName == ".idea" || baseName == ".vscode" + ) + ) || lib.hasSuffix ".iml" baseName + + # Filter out nix build files + || baseName == "Cargo.nix" + + # Filter out editor backup / swap files. + || lib.hasSuffix "~" baseName + || builtins.match "^\\.sw[a-z]$$" baseName != null + || builtins.match "^\\..*\\.sw[a-z]$$" baseName != null + || lib.hasSuffix ".tmp" baseName + || lib.hasSuffix ".bak" baseName + || baseName == "tests.nix" + ); + + /* Returns a crate which depends on successful test execution + of crate given as the second argument. + + testCrateFlags: list of flags to pass to the test exectuable + testInputs: list of packages that should be available during test execution + */ + crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs - else - pkgs: (buildRustCrateForPkgs pkgs).override { - defaultCrateOverrides = crateOverrides; - } + _: { + buildTests = true; + } ); - builtRustCrates = builtRustCratesWithFeatures { - inherit packageId features; - buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; - runTests = false; + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" + (pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ]); + in + pkgs.stdenvNoCC.mkDerivation { + name = "run-tests-${testCrate.name}"; + + inherit (crate) src; + + inherit testCrateFlags; + + buildInputs = testInputs; + + buildPhase = '' + set -e + export RUST_BACKTRACE=1 + + # build outputs + testRoot=target/debug + mkdir -p $testRoot + + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . + + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; }; - builtTestRustCrates = builtRustCratesWithFeatures { - inherit packageId features; - buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; - runTests = true; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; }; - drv = builtRustCrates.crates.${packageId}; - testDrv = builtTestRustCrates.crates.${packageId}; - derivation = - if runTests then - crateWithTest - { - crate = drv; - testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; - } - else drv; - in - derivation - ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; - - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc - for the corresponding crate. - */ - builtRustCratesWithFeatures = - { packageId - , features - , crateConfigs ? crates - , buildRustCrateForPkgsFunc - , runTests - , makeTarget ? makeDefaultTarget - } @ args: - assert (builtins.isAttrs crateConfigs); - assert (builtins.isString packageId); - assert (builtins.isList features); - assert (builtins.isAttrs (makeTarget stdenv.hostPlatform)); - assert (builtins.isBool runTests); - let - rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); - # Memoize built packages so that reappearing packages are only built once. - builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + } + (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + ''); + + /* A restricted overridable version of builtRustCratesWithFeatures. */ + buildRustCrateWithFeatures = + { packageId + , features ? rootFeatures + , crateOverrides ? defaultCrateOverrides + , buildRustCrateForPkgsFunc ? null + , runTests ? false + , testCrateFlags ? [ ] + , testInputs ? [ ] + # Any command to run immediatelly before a test is executed. + , testPreRun ? "" + # Any command run immediatelly after a test is executed. + , testPostRun ? "" + }: + lib.makeOverridable + ( + { features + , crateOverrides + , runTests + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + }: let - self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; - target = makeTarget stdenv.hostPlatform; - build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; + buildRustCrateForPkgsFuncOverriden = + if buildRustCrateForPkgsFunc != null + then buildRustCrateForPkgsFunc + else + ( + if crateOverrides == pkgs.defaultCrateOverrides + then buildRustCrateForPkgs + else + pkgs: (buildRustCrateForPkgs pkgs).override { + defaultCrateOverrides = crateOverrides; + } + ); + builtRustCrates = builtRustCratesWithFeatures { + inherit packageId features; + buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; + runTests = false; + }; + builtTestRustCrates = builtRustCratesWithFeatures { + inherit packageId features; + buildRustCrateForPkgsFunc = buildRustCrateForPkgsFuncOverriden; + runTests = true; }; + drv = builtRustCrates.crates.${packageId}; + testDrv = builtTestRustCrates.crates.${packageId}; + derivation = + if runTests then + crateWithTest + { + crate = drv; + testCrate = testDrv; + inherit testCrateFlags testInputs testPreRun testPostRun; + } + else drv; in - self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: - let - features = mergedFeatures."${packageId}" or [ ]; - crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; + derivation + ) + { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + + /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + for the corresponding crate. + */ + builtRustCratesWithFeatures = + { packageId + , features + , crateConfigs ? crates + , buildRustCrateForPkgsFunc + , runTests + , makeTarget ? makeDefaultTarget + } @ args: + assert (builtins.isAttrs crateConfigs); + assert (builtins.isString packageId); + assert (builtins.isList features); + assert (builtins.isAttrs (makeTarget stdenv.hostPlatform)); + assert (builtins.isBool runTests); + let + rootPackageId = packageId; + mergedFeatures = mergePackageFeatures + ( + args // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { test = runTests; }; + } + ); + # Memoize built packages so that reappearing packages are only built once. + builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; + mkBuiltByPackageIdByPkgs = pkgs: + let + self = { + crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + target = makeTarget stdenv.hostPlatform; + build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; - dependenciesWithRenames = - let - buildDeps = filterEnabledDependencies { + in + self; + buildByPackageIdForPkgsImpl = self: pkgs: packageId: + let + features = mergedFeatures."${packageId}" or [ ]; + crateConfig' = crateConfigs."${packageId}"; + crateConfig = + builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; + devDependencies = + lib.optionals + (runTests && packageId == rootPackageId) + (crateConfig'.devDependencies or [ ]); + dependencies = + dependencyDerivations { inherit features; inherit (self) target; - dependencies = crateConfig.dependencies or [ ] ++ devDependencies; + buildByPackageId = depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false + then self.build.crates.${depPackageId} + else self.crates.${depPackageId}; + dependencies = + (crateConfig.dependencies or [ ]) + ++ devDependencies; }; - hostDeps = filterEnabledDependencies { + buildDependencies = + dependencyDerivations { inherit features; inherit (self.build) target; + buildByPackageId = depPackageId: + self.build.crates.${depPackageId}; dependencies = crateConfig.buildDependencies or [ ]; }; - in - lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); - # Crate renames have the form: - # - # { - # crate_name = [ - # { version = "1.2.3"; rename = "crate_name01"; } - # ]; - # # ... - # } - crateRenames = - let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: - let - package = crateConfigs."${dep.packageId}"; - in - { inherit (dep) rename; inherit (package) version; }; - in - lib.mapAttrs (name: builtins.map versionAndRename) grouped; - in - buildRustCrateForPkgsFunc pkgs + dependenciesWithRenames = + let + buildDeps = filterEnabledDependencies { + inherit features; + inherit (self) target; + dependencies = crateConfig.dependencies or [ ] ++ devDependencies; + }; + hostDeps = filterEnabledDependencies { + inherit features; + inherit (self.build) target; + dependencies = crateConfig.buildDependencies or [ ]; + }; + in + lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); + # Crate renames have the form: + # + # { + # crate_name = [ + # { version = "1.2.3"; rename = "crate_name01"; } + # ]; + # # ... + # } + crateRenames = + let + grouped = + lib.groupBy + (dependency: dependency.name) + dependenciesWithRenames; + versionAndRename = dep: + let + package = crateConfigs."${dep.packageId}"; + in + { inherit (dep) rename; inherit (package) version; }; + in + lib.mapAttrs (name: builtins.map versionAndRename) grouped; + in + buildRustCrateForPkgsFunc pkgs + ( + crateConfig // { + src = crateConfig.src or ( + pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + } + ); + extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit features dependencies buildDependencies crateRenames release; + } + ); + in + builtByPackageIdByPkgs; + + /* Returns the actual derivations for the given dependencies. */ + dependencyDerivations = + { buildByPackageId + , features + , dependencies + , target + }: + assert (builtins.isList features); + assert (builtins.isList dependencies); + assert (builtins.isAttrs target); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies features target; + }; + depDerivation = dependency: buildByPackageId dependency.packageId; + in + map depDerivation enabledDependencies; + + /* Returns a sanitized version of val with all values substituted that cannot + be serialized as JSON. + */ + sanitizeForJson = val: + if builtins.isAttrs val + then lib.mapAttrs (n: sanitizeForJson) val + else if builtins.isList val + then builtins.map sanitizeForJson val + else if builtins.isFunction val + then "function" + else val; + + /* Returns various tools to debug a crate. */ + debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: + assert (builtins.isString packageId); + let + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; + buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; } ); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; + }; in - builtByPackageIdByPkgs; - - /* Returns the actual derivations for the given dependencies. */ - dependencyDerivations = - { buildByPackageId - , features - , dependencies - , target - }: - assert (builtins.isList features); + { internal = debug; }; + + /* Returns differences between cargo default features and crate2nix default + features. + + This is useful for verifying the feature resolution in crate2nix. + */ + diffDefaultPackageFeatures = + { crateConfigs ? crates + , packageId + , target + }: + assert (builtins.isAttrs crateConfigs); + let + prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); + mergedFeatures = + prefixValues + "crate2nix" + (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + configs = prefixValues "cargo" crateConfigs; + combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; + onlyInCargo = + builtins.attrNames + (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); + onlyInCrate2Nix = + builtins.attrNames + (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + differentFeatures = lib.filterAttrs + ( + n: v: + (v ? "crate2nix") + && (v ? "cargo") + && (v.crate2nix.features or [ ]) != (v."cargo".resolved_default_features or [ ]) + ) + combined; + in + builtins.toJSON { + inherit onlyInCargo onlyInCrate2Nix differentFeatures; + }; + + /* Returns an attrset mapping packageId to the list of enabled features. + + If multiple paths to a dependency enable different features, the + corresponding feature sets are merged. Features in rust are additive. + */ + mergePackageFeatures = + { crateConfigs ? crates + , packageId + , rootPackageId ? packageId + , features ? rootFeatures + , dependencyPath ? [ crates.${packageId}.crateName ] + , featuresByPackageId ? { } + , target + # Adds devDependencies to the crate with rootPackageId. + , runTests ? false + , ... + } @ args: + assert (builtins.isAttrs crateConfigs); + assert (builtins.isString packageId); + assert (builtins.isString rootPackageId); + assert (builtins.isList features); + assert (builtins.isList dependencyPath); + assert (builtins.isAttrs featuresByPackageId); + assert (builtins.isAttrs target); + assert (builtins.isBool runTests); + let + crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); + expandedFeatures = expandFeatures (crateConfig.features or { }) features; + enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; + depWithResolvedFeatures = dependency: + let + inherit (dependency) packageId; + features = dependencyFeatures enabledFeatures dependency; + in + { inherit packageId features; }; + resolveDependencies = cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache + ( + cache: { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures + then cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit crateConfigs packageId target runTests rootPackageId; + } + ); + cacheWithSelf = + let + cacheFeatures = featuresByPackageId.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); + in + featuresByPackageId // { + "${packageId}" = combinedFeatures; + }; + cacheWithDependencies = + resolveDependencies cacheWithSelf "dep" + ( + crateConfig.dependencies or [ ] + ++ lib.optionals + (runTests && packageId == rootPackageId) + (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = + resolveDependencies + cacheWithDependencies "build" + (crateConfig.buildDependencies or [ ]); + in + cacheWithAll; + + /* Returns the enabled dependencies given the enabled features. */ + filterEnabledDependencies = { dependencies, features, target }: assert (builtins.isList dependencies); + assert (builtins.isList features); assert (builtins.isAttrs target); + + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && ( + !(dep.optional or false) + || builtins.any (doesFeatureEnableDependency dep) features + ) + ) + dependencies; + + /* Returns whether the given feature should enable the given dependency. */ + doesFeatureEnableDependency = dependency: feature: let - enabledDependencies = filterEnabledDependencies { - inherit dependencies features target; - }; - depDerivation = dependency: buildByPackageId dependency.packageId; + name = dependency.rename or dependency.name; + prefix = "${name}/"; + len = builtins.stringLength prefix; + startsWithPrefix = builtins.substring 0 len feature == prefix; in - map depDerivation enabledDependencies; - - /* Returns a sanitized version of val with all values substituted that cannot - be serialized as JSON. - */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: sanitizeForJson) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; - }; - }; - in - { internal = debug; }; - - /* Returns differences between cargo default features and crate2nix default - features. - - This is useful for verifying the feature resolution in crate2nix. - */ - diffDefaultPackageFeatures = - { crateConfigs ? crates - , packageId - , target - }: - assert (builtins.isAttrs crateConfigs); + feature == name || feature == "dep:" + name || startsWithPrefix; + + /* Returns the expanded features for the given inputFeatures by applying the + rules in featureMap. + + featureMap is an attribute set which maps feature names to lists of further + feature names to enable in case this feature is selected. + */ + expandFeatures = featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = oldSeen: inputFeatures: + if inputFeatures != [ ] + then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { ${feature} = 1; }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; + + /* This function adds optional dependencies as features if they are enabled + indirectly by dependency features. This function mimics Cargo's behavior + described in a note at: + https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features + */ + enableFeatures = dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); let - prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); - configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); - differentFeatures = lib.filterAttrs + additionalFeatures = lib.concatMap ( - n: v: - (v ? "crate2nix") - && (v ? "cargo") - && (v.crate2nix.features or [ ]) != (v."cargo".resolved_default_features or [ ]) + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled + then [ (dependency.rename or dependency.name) ] + else [ ] ) - combined; + dependencies; in - builtins.toJSON { - inherit onlyInCargo onlyInCrate2Nix differentFeatures; - }; - - /* Returns an attrset mapping packageId to the list of enabled features. - - If multiple paths to a dependency enable different features, the - corresponding feature sets are merged. Features in rust are additive. - */ - mergePackageFeatures = - { crateConfigs ? crates - , packageId - , rootPackageId ? packageId - , features ? rootFeatures - , dependencyPath ? [ crates.${packageId}.crateName ] - , featuresByPackageId ? { } - , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false - , ... - } @ args: - assert (builtins.isAttrs crateConfigs); - assert (builtins.isString packageId); - assert (builtins.isString rootPackageId); + sortedUnique (features ++ additionalFeatures); + + /* + Returns the actual features for the given dependency. + + features: The features of the crate that refers this dependency. + */ + dependencyFeatures = features: dependency: assert (builtins.isList features); - assert (builtins.isList dependencyPath); - assert (builtins.isAttrs featuresByPackageId); - assert (builtins.isAttrs target); - assert (builtins.isBool runTests); + assert (builtins.isAttrs dependency); let - crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); - expandedFeatures = expandFeatures (crateConfig.features or { }) features; - enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: - let - inherit (dependency) packageId; - features = dependencyFeatures enabledFeatures dependency; - in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); + defaultOrNil = + if dependency.usesDefaultFeatures or true + then [ "default" ] + else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } - ); - cacheWithSelf = - let - cacheFeatures = featuresByPackageId.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: + if lib.hasPrefix prefix s + then lib.removePrefix prefix s + else null; + extractFeature = feature: lib.findFirst + (f: f != null) + null + (map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ]); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); in - featuresByPackageId // { - "${packageId}" = combinedFeatures; - }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + dependencyFeatures; in - cacheWithAll; + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + /* Sorts and removes duplicates from a list of strings. */ + sortedUnique = features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features - ) - ) - dependencies; - - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: - let - name = dependency.rename or dependency.name; - prefix = "${name}/"; - len = builtins.stringLength prefix; - startsWithPrefix = builtins.substring 0 len feature == prefix; - in - feature == name || feature == "dep:" + name || startsWithPrefix; - - /* Returns the expanded features for the given inputFeatures by applying the - rules in featureMap. - - featureMap is an attribute set which maps feature names to lists of further - feature names to enable in case this feature is selected. - */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; - - /* This function adds optional dependencies as features if they are enabled - indirectly by dependency features. This function mimics Cargo's behavior - described in a note at: - https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features - */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); - - /* - Returns the actual features for the given dependency. - - features: The features of the crate that refers this dependency. - */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; - - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + deprecationWarning = message: value: + if strictDeprecation + then builtins.throw "strictDeprecation enabled, aborting: ${message}" + else builtins.trace message value; - # - # crate2nix/default.nix (excerpt end) - # + # + # crate2nix/default.nix (excerpt end) + # }; } diff --git a/crate2nix/src/lib.rs b/crate2nix/src/lib.rs index 03232a01..1d3a83ef 100644 --- a/crate2nix/src/lib.rs +++ b/crate2nix/src/lib.rs @@ -221,7 +221,10 @@ fn prefetch_and_fill_crates_sha256( } /// Prefetch hashes when necessary. -fn prefetch_and_fill_registries(config: &GenerateConfig, default_nix: &mut BuildInfo) -> Result<(), Error> { +fn prefetch_and_fill_registries( + config: &GenerateConfig, + default_nix: &mut BuildInfo, +) -> Result<(), Error> { default_nix.registries = prefetch::prefetch_registries(config, &mut default_nix.crates) .map_err(|e| format_err!("while prefetching crates for calculating sha256: {}", e))?; diff --git a/crate2nix/src/prefetch.rs b/crate2nix/src/prefetch.rs index 053ad0d2..14ea9e32 100644 --- a/crate2nix/src/prefetch.rs +++ b/crate2nix/src/prefetch.rs @@ -177,7 +177,6 @@ pub fn prefetch_registries( config: &GenerateConfig, crate_derivations: &[CrateDerivation], ) -> Result, Error> { - let hashes_string: String = if config.read_crate_hashes { std::fs::read_to_string(&config.registry_hashes_json).unwrap_or_else(|_| "{}".to_string()) } else { @@ -307,8 +306,8 @@ impl PrefetchableSource for RegistrySource { } fn prefetch(&self) -> Result { - // This is done in two steps, currently only implemented in - // the generated Nix. + // This is done in two steps, currently only implemented in + // the generated Nix. unimplemented!() } } diff --git a/crate2nix/templates/nix/crate2nix/default.nix b/crate2nix/templates/nix/crate2nix/default.nix index c844c03a..3e088032 100644 --- a/crate2nix/templates/nix/crate2nix/default.nix +++ b/crate2nix/templates/nix/crate2nix/default.nix @@ -5,20 +5,23 @@ , lib , stdenv , buildRustCrate -, buildRustCrateForPkgs ? if buildRustCrate != null - then lib.warn "`buildRustCrate` is deprecated, use `buildRustCrateForPkgs` instead" (_: buildRustCrate) - else pkgs: pkgs.buildRustCrate +, buildRustCrateForPkgs ? if buildRustCrate != null then + lib.warn "`buildRustCrate` is deprecated, use `buildRustCrateForPkgs` instead" (_: buildRustCrate) + else + pkgs: pkgs.buildRustCrate , defaultCrateOverrides , strictDeprecation ? true , crates ? { } , rootFeatures ? [ ] , targetFeatures ? [ ] , release ? true +, }: rec { # #} - /* Target (platform) data for conditional dependencies. + /* + Target (platform) data for conditional dependencies. This corresponds roughly to what buildRustCrate is setting. */ makeDefaultTarget = platform: { @@ -30,50 +33,70 @@ rec { inherit (platform.rust.platform) arch os - vendor; + vendor + ; family = platform.rust.platform.target-family; env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; + endian = if platform.parsed.cpu.significantByte.name == "littleEndian" then "little" else "big"; pointer_width = toString platform.parsed.cpu.bits; debug_assertions = false; }; - registryUrl = { registries, url, crate, version, sha256 }: - let dl = registries.${url}.dl; - tmpl = [ "{crate}" "{version}" "{prefix}" "{lowerprefix}" "{sha256-checksum}" ]; + registryUrl = + { registries + , url + , crate + , version + , sha256 + , + }: + let + dl = registries.${url}.dl; + tmpl = [ + "{crate}" + "{version}" + "{prefix}" + "{lowerprefix}" + "{sha256-checksum}" + ]; in - with lib.strings; + with lib.strings; if lib.lists.any (i: hasInfix "{}" dl) tmpl then - let prefix = - if builtins.stringLength crate == 1 then - "1" - else if builtins.stringLength crate == 2 then - "2" - else - "${builtins.substring 0 2 crate}/${builtins.substring 2 (builtins.stringLength crate - 2) crate}"; + let + prefix = + if builtins.stringLength crate == 1 then + "1" + else if builtins.stringLength crate == 2 then + "2" + else + "${builtins.substring 0 2 crate}/${builtins.substring 2 (builtins.stringLength crate - 2) crate}"; in - builtins.replaceStrings tmpl [ - crate version prefix (lib.strings.toLower prefix) sha256 - ] + builtins.replaceStrings tmpl [ + crate + version + prefix + (lib.strings.toLower prefix) + sha256 + ] else "${dl}/${crate}/${version}/download"; - /* Filters common temp files and build files. */ + # Filters common temp files and build files. # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: + sourceFilter = + name: type: let baseName = builtins.baseNameOf (builtins.toString name); in - ! ( + !( # Filter out git baseName == ".gitignore" || (type == "directory" && baseName == ".git") # Filter out build results || ( - type == "directory" && ( + type == "directory" + && ( baseName == "target" || baseName == "_site" || baseName == ".sass-cache" @@ -83,16 +106,11 @@ rec { ) # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + || (type == "symlink" && lib.hasPrefix "result" baseName) # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" - ) - ) || lib.hasSuffix ".iml" baseName + || (type == "directory" && (baseName == ".idea" || baseName == ".vscode")) + || lib.hasSuffix ".iml" baseName # Filter out nix build files || baseName == "Cargo.nix" @@ -106,90 +124,100 @@ rec { || baseName == "tests.nix" ); - /* Returns a crate which depends on successful test execution + /* + Returns a crate which depends on successful test execution of crate given as the second argument. testCrateFlags: list of flags to pass to the test exectuable testInputs: list of packages that should be available during test execution */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - } + crateWithTest = + { crate + , testCrate + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + , + }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override (_: { + buildTests = true; + }); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" ( + pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ] ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.stdenvNoCC.mkDerivation { - name = "run-tests-${testCrate.name}"; - - inherit (crate) src; - - inherit testCrateFlags; - - buildInputs = testInputs; - - buildPhase = '' - set -e - export RUST_BACKTRACE=1 - - # build outputs - testRoot=target/debug - mkdir -p $testRoot - - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . - - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - }; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); - - /* A restricted overridable version of builtRustCratesWithFeatures. */ + in + pkgs.stdenvNoCC.mkDerivation { + name = "run-tests-${testCrate.name}"; + + inherit (crate) src; + + inherit testCrateFlags; + + buildInputs = testInputs; + + buildPhase = '' + set -e + export RUST_BACKTRACE=1 + + # build outputs + testRoot=target/debug + mkdir -p $testRoot + + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . + + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + }; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; + }; + } + ( + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + '' + ); + + # A restricted overridable version of builtRustCratesWithFeatures. buildRustCrateWithFeatures = { packageId , features ? rootFeatures @@ -198,10 +226,11 @@ rec { , runTests ? false , testCrateFlags ? [ ] , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" + , # Any command to run immediatelly before a test is executed. + testPreRun ? "" + , # Any command run immediatelly after a test is executed. + testPostRun ? "" + , }: lib.makeOverridable ( @@ -212,17 +241,19 @@ rec { , testInputs , testPreRun , testPostRun + , }: let buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc + if buildRustCrateForPkgsFunc != null then + buildRustCrateForPkgsFunc else ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs + if crateOverrides == pkgs.defaultCrateOverrides then + buildRustCrateForPkgs else - pkgs: (buildRustCrateForPkgs pkgs).override { + pkgs: + (buildRustCrateForPkgs pkgs).override { defaultCrateOverrides = crateOverrides; } ); @@ -244,15 +275,32 @@ rec { { crate = drv; testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; + inherit + testCrateFlags + testInputs + testPreRun + testPostRun + ; } - else drv; + else + drv; in derivation ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + { + inherit + features + crateOverrides + runTests + testCrateFlags + testInputs + testPreRun + testPostRun + ; + }; - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + /* + Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc for the corresponding crate. */ builtRustCratesWithFeatures = @@ -262,7 +310,8 @@ rec { , buildRustCrateForPkgsFunc , runTests , makeTarget ? makeDefaultTarget - } @ args: + , + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isList features); @@ -270,55 +319,61 @@ rec { assert (builtins.isBool runTests); let rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); + mergedFeatures = mergePackageFeatures ( + args + // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { + test = runTests; + }; + } + ); # Memoize built packages so that reappearing packages are only built once. builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + mkBuiltByPackageIdByPkgs = + pkgs: let self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + crates = lib.mapAttrs + ( + packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId + ) + crateConfigs; target = makeTarget stdenv.hostPlatform; build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: + buildByPackageIdForPkgsImpl = + self: pkgs: packageId: let features = mergedFeatures."${packageId}" or [ ]; crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; - }; + crateConfig = builtins.removeAttrs crateConfig' [ + "resolvedDefaultFeatures" + "devDependencies" + ]; + devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( + crateConfig'.devDependencies or [ ] + ); + dependencies = dependencyDerivations { + inherit features; + inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; + dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; + }; + buildDependencies = dependencyDerivations { + inherit features; + inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; + dependencies = crateConfig.buildDependencies or [ ]; + }; dependenciesWithRenames = let buildDeps = filterEnabledDependencies { @@ -343,45 +398,54 @@ rec { # } crateRenames = let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: + grouped = lib.groupBy (dependency: dependency.name) dependenciesWithRenames; + versionAndRename = + dep: let package = crateConfigs."${dep.packageId}"; in - { inherit (dep) rename; inherit (package) version; }; + { + inherit (dep) rename; + inherit (package) version; + }; in lib.mapAttrs (name: builtins.map versionAndRename) grouped; in - buildRustCrateForPkgsFunc pkgs - ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; - } - ); + buildRustCrateForPkgsFunc pkgs ( + crateConfig + // { + src = + crateConfig.src or (pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + }); + extraRustcOpts = + lib.lists.optional (targetFeatures != [ ]) + "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit + features + dependencies + buildDependencies + crateRenames + release + ; + } + ); in builtByPackageIdByPkgs; - /* Returns the actual derivations for the given dependencies. */ + # Returns the actual derivations for the given dependencies. dependencyDerivations = { buildByPackageId , features , dependencies , target + , }: assert (builtins.isList features); assert (builtins.isList dependencies); @@ -394,52 +458,59 @@ rec { in map depDerivation enabledDependencies; - /* Returns a sanitized version of val with all values substituted that cannot + /* + Returns a sanitized version of val with all values substituted that cannot be serialized as JSON. */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: sanitizeForJson) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; + sanitizeForJson = + val: + if builtins.isAttrs val then + lib.mapAttrs (n: sanitizeForJson) val + else if builtins.isList val then + builtins.map sanitizeForJson val + else if builtins.isFunction val then + "function" + else + val; + + # Returns various tools to debug a crate. + debugCrate = + { packageId + , target ? makeDefaultTarget stdenv.hostPlatform + , + }: + assert (builtins.isString packageId); + let + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson (buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; + }); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; }; + in + { + internal = debug; }; - in - { internal = debug; }; - /* Returns differences between cargo default features and crate2nix default + /* + Returns differences between cargo default features and crate2nix default features. This is useful for verifying the feature resolution in crate2nix. @@ -448,22 +519,26 @@ rec { { crateConfigs ? crates , packageId , target + , }: assert (builtins.isAttrs crateConfigs); let prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + mergedFeatures = prefixValues "crate2nix" (mergePackageFeatures { + inherit crateConfigs packageId target; + features = [ "default" ]; + }); configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + combined = lib.foldAttrs (a: b: a // b) { } [ + mergedFeatures + configs + ]; + onlyInCargo = builtins.attrNames ( + lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined + ); + onlyInCrate2Nix = builtins.attrNames ( + lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined + ); differentFeatures = lib.filterAttrs ( n: v: @@ -477,7 +552,8 @@ rec { inherit onlyInCargo onlyInCrate2Nix differentFeatures; }; - /* Returns an attrset mapping packageId to the list of enabled features. + /* + Returns an attrset mapping packageId to the list of enabled features. If multiple paths to a dependency enable different features, the corresponding feature sets are merged. Features in rust are additive. @@ -490,10 +566,10 @@ rec { , dependencyPath ? [ crates.${packageId}.crateName ] , featuresByPackageId ? { } , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false + , # Adds devDependencies to the crate with rootPackageId. + runTests ? false , ... - } @ args: + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isString rootPackageId); @@ -506,84 +582,93 @@ rec { crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); expandedFeatures = expandFeatures (crateConfig.features or { }) features; enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: + depWithResolvedFeatures = + dependency: let inherit (dependency) packageId; features = dependencyFeatures enabledFeatures dependency; in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } + { + inherit packageId features; + }; + resolveDependencies = + cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache ( + cache: + { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then + cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit + crateConfigs + packageId + target + runTests + rootPackageId + ; + } ); cacheWithSelf = let cacheFeatures = featuresByPackageId.${packageId} or [ ]; combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in - featuresByPackageId // { + featuresByPackageId + // { "${packageId}" = combinedFeatures; }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + cacheWithDependencies = resolveDependencies cacheWithSelf "dep" ( + crateConfig.dependencies or [ ] + ++ lib.optionals (runTests && packageId == rootPackageId) (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = resolveDependencies cacheWithDependencies "build" ( + crateConfig.buildDependencies or [ ] + ); in cacheWithAll; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + # Returns the enabled dependencies given the enabled features. + filterEnabledDependencies = + { dependencies + , features + , target + , + }: + assert (builtins.isList dependencies); + assert (builtins.isList features); + assert (builtins.isAttrs target); - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) - ) - dependencies; + dependencies; - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: + # Returns whether the given feature should enable the given dependency. + doesFeatureEnableDependency = + dependency: feature: let name = dependency.rename or dependency.name; prefix = "${name}/"; @@ -592,109 +677,116 @@ rec { in feature == name || feature == "dep:" + name || startsWithPrefix; - /* Returns the expanded features for the given inputFeatures by applying the + /* + Returns the expanded features for the given inputFeatures by applying the rules in featureMap. featureMap is an attribute set which maps feature names to lists of further feature names to enable in case this feature is selected. */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; + expandFeatures = + featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; - /* This function adds optional dependencies as features if they are enabled + /* + This function adds optional dependencies as features if they are enabled indirectly by dependency features. This function mimics Cargo's behavior described in a note at: https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); + enableFeatures = + dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); + let + additionalFeatures = lib.concatMap + ( + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled then + [ (dependency.rename or dependency.name) ] + else + [ ] + ) + dependencies; + in + sortedUnique (features ++ additionalFeatures); /* Returns the actual features for the given dependency. features: The features of the crate that refers this dependency. */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; + dependencyFeatures = + features: dependency: + assert (builtins.isList features); + assert (builtins.isAttrs dependency); + let + defaultOrNil = if dependency.usesDefaultFeatures or true then [ "default" ] else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = + let + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: if lib.hasPrefix prefix s then lib.removePrefix prefix s else null; + extractFeature = + feature: + lib.findFirst (f: f != null) null ( + map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ] + ); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); + in + dependencyFeatures; + in + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + # Sorts and removes duplicates from a list of strings. + sortedUnique = + features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + deprecationWarning = + message: value: + if strictDeprecation then + builtins.throw "strictDeprecation enabled, aborting: ${message}" + else + builtins.trace message value; # # crate2nix/default.nix (excerpt end) diff --git a/sample_projects/bin_with_git_submodule_dep/Cargo.nix b/sample_projects/bin_with_git_submodule_dep/Cargo.nix index e731beef..34060349 100644 --- a/sample_projects/bin_with_git_submodule_dep/Cargo.nix +++ b/sample_projects/bin_with_git_submodule_dep/Cargo.nix @@ -5,6 +5,7 @@ { nixpkgs ? , pkgs ? import nixpkgs { config = {}; } +, fetchurl ? pkgs.fetchurl , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? pkgs: pkgs.buildRustCrate @@ -61,6 +62,8 @@ rec { }; }; + + # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { name = "all-workspace-members"; @@ -1364,7 +1367,8 @@ rec { # crate2nix/default.nix (excerpt start) # - /* Target (platform) data for conditional dependencies. + /* + Target (platform) data for conditional dependencies. This corresponds roughly to what buildRustCrate is setting. */ makeDefaultTarget = platform: { @@ -1376,30 +1380,70 @@ rec { inherit (platform.rust.platform) arch os - vendor; + vendor + ; family = platform.rust.platform.target-family; env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; + endian = if platform.parsed.cpu.significantByte.name == "littleEndian" then "little" else "big"; pointer_width = toString platform.parsed.cpu.bits; debug_assertions = false; }; - /* Filters common temp files and build files. */ + registryUrl = + { registries + , url + , crate + , version + , sha256 + , + }: + let + dl = registries.${url}.dl; + tmpl = [ + "{crate}" + "{version}" + "{prefix}" + "{lowerprefix}" + "{sha256-checksum}" + ]; + in + with lib.strings; + if lib.lists.any (i: hasInfix "{}" dl) tmpl then + let + prefix = + if builtins.stringLength crate == 1 then + "1" + else if builtins.stringLength crate == 2 then + "2" + else + "${builtins.substring 0 2 crate}/${builtins.substring 2 (builtins.stringLength crate - 2) crate}"; + in + builtins.replaceStrings tmpl [ + crate + version + prefix + (lib.strings.toLower prefix) + sha256 + ] + else + "${dl}/${crate}/${version}/download"; + + # Filters common temp files and build files. # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: + sourceFilter = + name: type: let baseName = builtins.baseNameOf (builtins.toString name); in - ! ( + !( # Filter out git baseName == ".gitignore" || (type == "directory" && baseName == ".git") # Filter out build results || ( - type == "directory" && ( + type == "directory" + && ( baseName == "target" || baseName == "_site" || baseName == ".sass-cache" @@ -1409,16 +1453,11 @@ rec { ) # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + || (type == "symlink" && lib.hasPrefix "result" baseName) # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" - ) - ) || lib.hasSuffix ".iml" baseName + || (type == "directory" && (baseName == ".idea" || baseName == ".vscode")) + || lib.hasSuffix ".iml" baseName # Filter out nix build files || baseName == "Cargo.nix" @@ -1432,90 +1471,100 @@ rec { || baseName == "tests.nix" ); - /* Returns a crate which depends on successful test execution + /* + Returns a crate which depends on successful test execution of crate given as the second argument. testCrateFlags: list of flags to pass to the test exectuable testInputs: list of packages that should be available during test execution */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - } + crateWithTest = + { crate + , testCrate + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + , + }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override (_: { + buildTests = true; + }); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" ( + pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ] ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.stdenvNoCC.mkDerivation { - name = "run-tests-${testCrate.name}"; - - inherit (crate) src; - - inherit testCrateFlags; - - buildInputs = testInputs; - - buildPhase = '' - set -e - export RUST_BACKTRACE=1 - - # build outputs - testRoot=target/debug - mkdir -p $testRoot - - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . - - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - }; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); - - /* A restricted overridable version of builtRustCratesWithFeatures. */ + in + pkgs.stdenvNoCC.mkDerivation { + name = "run-tests-${testCrate.name}"; + + inherit (crate) src; + + inherit testCrateFlags; + + buildInputs = testInputs; + + buildPhase = '' + set -e + export RUST_BACKTRACE=1 + + # build outputs + testRoot=target/debug + mkdir -p $testRoot + + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . + + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + }; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; + }; + } + ( + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + '' + ); + + # A restricted overridable version of builtRustCratesWithFeatures. buildRustCrateWithFeatures = { packageId , features ? rootFeatures @@ -1524,10 +1573,11 @@ rec { , runTests ? false , testCrateFlags ? [ ] , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" + , # Any command to run immediatelly before a test is executed. + testPreRun ? "" + , # Any command run immediatelly after a test is executed. + testPostRun ? "" + , }: lib.makeOverridable ( @@ -1538,17 +1588,19 @@ rec { , testInputs , testPreRun , testPostRun + , }: let buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc + if buildRustCrateForPkgsFunc != null then + buildRustCrateForPkgsFunc else ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs + if crateOverrides == pkgs.defaultCrateOverrides then + buildRustCrateForPkgs else - pkgs: (buildRustCrateForPkgs pkgs).override { + pkgs: + (buildRustCrateForPkgs pkgs).override { defaultCrateOverrides = crateOverrides; } ); @@ -1570,15 +1622,32 @@ rec { { crate = drv; testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; + inherit + testCrateFlags + testInputs + testPreRun + testPostRun + ; } - else drv; + else + drv; in derivation ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + { + inherit + features + crateOverrides + runTests + testCrateFlags + testInputs + testPreRun + testPostRun + ; + }; - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + /* + Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc for the corresponding crate. */ builtRustCratesWithFeatures = @@ -1588,7 +1657,8 @@ rec { , buildRustCrateForPkgsFunc , runTests , makeTarget ? makeDefaultTarget - } @ args: + , + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isList features); @@ -1596,55 +1666,61 @@ rec { assert (builtins.isBool runTests); let rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); + mergedFeatures = mergePackageFeatures ( + args + // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { + test = runTests; + }; + } + ); # Memoize built packages so that reappearing packages are only built once. builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + mkBuiltByPackageIdByPkgs = + pkgs: let self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + crates = lib.mapAttrs + ( + packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId + ) + crateConfigs; target = makeTarget stdenv.hostPlatform; build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: + buildByPackageIdForPkgsImpl = + self: pkgs: packageId: let features = mergedFeatures."${packageId}" or [ ]; crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; - }; + crateConfig = builtins.removeAttrs crateConfig' [ + "resolvedDefaultFeatures" + "devDependencies" + ]; + devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( + crateConfig'.devDependencies or [ ] + ); + dependencies = dependencyDerivations { + inherit features; + inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; + dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; + }; + buildDependencies = dependencyDerivations { + inherit features; + inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; + dependencies = crateConfig.buildDependencies or [ ]; + }; dependenciesWithRenames = let buildDeps = filterEnabledDependencies { @@ -1669,45 +1745,54 @@ rec { # } crateRenames = let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: + grouped = lib.groupBy (dependency: dependency.name) dependenciesWithRenames; + versionAndRename = + dep: let package = crateConfigs."${dep.packageId}"; in - { inherit (dep) rename; inherit (package) version; }; + { + inherit (dep) rename; + inherit (package) version; + }; in lib.mapAttrs (name: builtins.map versionAndRename) grouped; in - buildRustCrateForPkgsFunc pkgs - ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; - } - ); + buildRustCrateForPkgsFunc pkgs ( + crateConfig + // { + src = + crateConfig.src or (pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + }); + extraRustcOpts = + lib.lists.optional (targetFeatures != [ ]) + "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit + features + dependencies + buildDependencies + crateRenames + release + ; + } + ); in builtByPackageIdByPkgs; - /* Returns the actual derivations for the given dependencies. */ + # Returns the actual derivations for the given dependencies. dependencyDerivations = { buildByPackageId , features , dependencies , target + , }: assert (builtins.isList features); assert (builtins.isList dependencies); @@ -1720,52 +1805,59 @@ rec { in map depDerivation enabledDependencies; - /* Returns a sanitized version of val with all values substituted that cannot + /* + Returns a sanitized version of val with all values substituted that cannot be serialized as JSON. */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: sanitizeForJson) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; + sanitizeForJson = + val: + if builtins.isAttrs val then + lib.mapAttrs (n: sanitizeForJson) val + else if builtins.isList val then + builtins.map sanitizeForJson val + else if builtins.isFunction val then + "function" + else + val; + + # Returns various tools to debug a crate. + debugCrate = + { packageId + , target ? makeDefaultTarget stdenv.hostPlatform + , + }: + assert (builtins.isString packageId); + let + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson (buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; + }); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; }; + in + { + internal = debug; }; - in - { internal = debug; }; - /* Returns differences between cargo default features and crate2nix default + /* + Returns differences between cargo default features and crate2nix default features. This is useful for verifying the feature resolution in crate2nix. @@ -1774,22 +1866,26 @@ rec { { crateConfigs ? crates , packageId , target + , }: assert (builtins.isAttrs crateConfigs); let prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + mergedFeatures = prefixValues "crate2nix" (mergePackageFeatures { + inherit crateConfigs packageId target; + features = [ "default" ]; + }); configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + combined = lib.foldAttrs (a: b: a // b) { } [ + mergedFeatures + configs + ]; + onlyInCargo = builtins.attrNames ( + lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined + ); + onlyInCrate2Nix = builtins.attrNames ( + lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined + ); differentFeatures = lib.filterAttrs ( n: v: @@ -1803,7 +1899,8 @@ rec { inherit onlyInCargo onlyInCrate2Nix differentFeatures; }; - /* Returns an attrset mapping packageId to the list of enabled features. + /* + Returns an attrset mapping packageId to the list of enabled features. If multiple paths to a dependency enable different features, the corresponding feature sets are merged. Features in rust are additive. @@ -1816,10 +1913,10 @@ rec { , dependencyPath ? [ crates.${packageId}.crateName ] , featuresByPackageId ? { } , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false + , # Adds devDependencies to the crate with rootPackageId. + runTests ? false , ... - } @ args: + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isString rootPackageId); @@ -1832,84 +1929,93 @@ rec { crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); expandedFeatures = expandFeatures (crateConfig.features or { }) features; enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: + depWithResolvedFeatures = + dependency: let inherit (dependency) packageId; features = dependencyFeatures enabledFeatures dependency; in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } + { + inherit packageId features; + }; + resolveDependencies = + cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache ( + cache: + { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then + cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit + crateConfigs + packageId + target + runTests + rootPackageId + ; + } ); cacheWithSelf = let cacheFeatures = featuresByPackageId.${packageId} or [ ]; combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in - featuresByPackageId // { + featuresByPackageId + // { "${packageId}" = combinedFeatures; }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + cacheWithDependencies = resolveDependencies cacheWithSelf "dep" ( + crateConfig.dependencies or [ ] + ++ lib.optionals (runTests && packageId == rootPackageId) (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = resolveDependencies cacheWithDependencies "build" ( + crateConfig.buildDependencies or [ ] + ); in cacheWithAll; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + # Returns the enabled dependencies given the enabled features. + filterEnabledDependencies = + { dependencies + , features + , target + , + }: + assert (builtins.isList dependencies); + assert (builtins.isList features); + assert (builtins.isAttrs target); - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) - ) - dependencies; + dependencies; - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: + # Returns whether the given feature should enable the given dependency. + doesFeatureEnableDependency = + dependency: feature: let name = dependency.rename or dependency.name; prefix = "${name}/"; @@ -1918,109 +2024,116 @@ rec { in feature == name || feature == "dep:" + name || startsWithPrefix; - /* Returns the expanded features for the given inputFeatures by applying the + /* + Returns the expanded features for the given inputFeatures by applying the rules in featureMap. featureMap is an attribute set which maps feature names to lists of further feature names to enable in case this feature is selected. */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; + expandFeatures = + featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; - /* This function adds optional dependencies as features if they are enabled + /* + This function adds optional dependencies as features if they are enabled indirectly by dependency features. This function mimics Cargo's behavior described in a note at: https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); + enableFeatures = + dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); + let + additionalFeatures = lib.concatMap + ( + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled then + [ (dependency.rename or dependency.name) ] + else + [ ] + ) + dependencies; + in + sortedUnique (features ++ additionalFeatures); /* Returns the actual features for the given dependency. features: The features of the crate that refers this dependency. */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + dependencyFeatures = + features: dependency: + assert (builtins.isList features); + assert (builtins.isAttrs dependency); + let + defaultOrNil = if dependency.usesDefaultFeatures or true then [ "default" ] else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = + let + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: if lib.hasPrefix prefix s then lib.removePrefix prefix s else null; + extractFeature = + feature: + lib.findFirst (f: f != null) null ( + map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ] + ); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); + in + dependencyFeatures; + in + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + # Sorts and removes duplicates from a list of strings. + sortedUnique = + features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; + + deprecationWarning = + message: value: + if strictDeprecation then + builtins.throw "strictDeprecation enabled, aborting: ${message}" + else + builtins.trace message value; # # crate2nix/default.nix (excerpt end) diff --git a/sample_projects/codegen/Cargo.nix b/sample_projects/codegen/Cargo.nix index 8ff30078..91bdd375 100644 --- a/sample_projects/codegen/Cargo.nix +++ b/sample_projects/codegen/Cargo.nix @@ -5,6 +5,7 @@ { nixpkgs ? , pkgs ? import nixpkgs { config = {}; } +, fetchurl ? pkgs.fetchurl , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? pkgs: pkgs.buildRustCrate @@ -61,6 +62,8 @@ rec { }; }; + + # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { name = "all-workspace-members"; @@ -539,7 +542,8 @@ rec { # crate2nix/default.nix (excerpt start) # - /* Target (platform) data for conditional dependencies. + /* + Target (platform) data for conditional dependencies. This corresponds roughly to what buildRustCrate is setting. */ makeDefaultTarget = platform: { @@ -551,30 +555,70 @@ rec { inherit (platform.rust.platform) arch os - vendor; + vendor + ; family = platform.rust.platform.target-family; env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; + endian = if platform.parsed.cpu.significantByte.name == "littleEndian" then "little" else "big"; pointer_width = toString platform.parsed.cpu.bits; debug_assertions = false; }; - /* Filters common temp files and build files. */ + registryUrl = + { registries + , url + , crate + , version + , sha256 + , + }: + let + dl = registries.${url}.dl; + tmpl = [ + "{crate}" + "{version}" + "{prefix}" + "{lowerprefix}" + "{sha256-checksum}" + ]; + in + with lib.strings; + if lib.lists.any (i: hasInfix "{}" dl) tmpl then + let + prefix = + if builtins.stringLength crate == 1 then + "1" + else if builtins.stringLength crate == 2 then + "2" + else + "${builtins.substring 0 2 crate}/${builtins.substring 2 (builtins.stringLength crate - 2) crate}"; + in + builtins.replaceStrings tmpl [ + crate + version + prefix + (lib.strings.toLower prefix) + sha256 + ] + else + "${dl}/${crate}/${version}/download"; + + # Filters common temp files and build files. # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: + sourceFilter = + name: type: let baseName = builtins.baseNameOf (builtins.toString name); in - ! ( + !( # Filter out git baseName == ".gitignore" || (type == "directory" && baseName == ".git") # Filter out build results || ( - type == "directory" && ( + type == "directory" + && ( baseName == "target" || baseName == "_site" || baseName == ".sass-cache" @@ -584,16 +628,11 @@ rec { ) # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + || (type == "symlink" && lib.hasPrefix "result" baseName) # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" - ) - ) || lib.hasSuffix ".iml" baseName + || (type == "directory" && (baseName == ".idea" || baseName == ".vscode")) + || lib.hasSuffix ".iml" baseName # Filter out nix build files || baseName == "Cargo.nix" @@ -607,90 +646,100 @@ rec { || baseName == "tests.nix" ); - /* Returns a crate which depends on successful test execution + /* + Returns a crate which depends on successful test execution of crate given as the second argument. testCrateFlags: list of flags to pass to the test exectuable testInputs: list of packages that should be available during test execution */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - } + crateWithTest = + { crate + , testCrate + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + , + }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override (_: { + buildTests = true; + }); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" ( + pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ] ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.stdenvNoCC.mkDerivation { - name = "run-tests-${testCrate.name}"; + in + pkgs.stdenvNoCC.mkDerivation { + name = "run-tests-${testCrate.name}"; - inherit (crate) src; + inherit (crate) src; - inherit testCrateFlags; + inherit testCrateFlags; - buildInputs = testInputs; + buildInputs = testInputs; - buildPhase = '' - set -e - export RUST_BACKTRACE=1 + buildPhase = '' + set -e + export RUST_BACKTRACE=1 - # build outputs - testRoot=target/debug - mkdir -p $testRoot + # build outputs + testRoot=target/debug + mkdir -p $testRoot - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - }; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + }; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; + }; + } + ( + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + '' + ); - /* A restricted overridable version of builtRustCratesWithFeatures. */ + # A restricted overridable version of builtRustCratesWithFeatures. buildRustCrateWithFeatures = { packageId , features ? rootFeatures @@ -699,10 +748,11 @@ rec { , runTests ? false , testCrateFlags ? [ ] , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" + , # Any command to run immediatelly before a test is executed. + testPreRun ? "" + , # Any command run immediatelly after a test is executed. + testPostRun ? "" + , }: lib.makeOverridable ( @@ -713,17 +763,19 @@ rec { , testInputs , testPreRun , testPostRun + , }: let buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc + if buildRustCrateForPkgsFunc != null then + buildRustCrateForPkgsFunc else ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs + if crateOverrides == pkgs.defaultCrateOverrides then + buildRustCrateForPkgs else - pkgs: (buildRustCrateForPkgs pkgs).override { + pkgs: + (buildRustCrateForPkgs pkgs).override { defaultCrateOverrides = crateOverrides; } ); @@ -745,15 +797,32 @@ rec { { crate = drv; testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; + inherit + testCrateFlags + testInputs + testPreRun + testPostRun + ; } - else drv; + else + drv; in derivation ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + { + inherit + features + crateOverrides + runTests + testCrateFlags + testInputs + testPreRun + testPostRun + ; + }; - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + /* + Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc for the corresponding crate. */ builtRustCratesWithFeatures = @@ -763,7 +832,8 @@ rec { , buildRustCrateForPkgsFunc , runTests , makeTarget ? makeDefaultTarget - } @ args: + , + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isList features); @@ -771,55 +841,61 @@ rec { assert (builtins.isBool runTests); let rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); + mergedFeatures = mergePackageFeatures ( + args + // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { + test = runTests; + }; + } + ); # Memoize built packages so that reappearing packages are only built once. builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + mkBuiltByPackageIdByPkgs = + pkgs: let self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + crates = lib.mapAttrs + ( + packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId + ) + crateConfigs; target = makeTarget stdenv.hostPlatform; build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: + buildByPackageIdForPkgsImpl = + self: pkgs: packageId: let features = mergedFeatures."${packageId}" or [ ]; crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; - }; + crateConfig = builtins.removeAttrs crateConfig' [ + "resolvedDefaultFeatures" + "devDependencies" + ]; + devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( + crateConfig'.devDependencies or [ ] + ); + dependencies = dependencyDerivations { + inherit features; + inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; + dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; + }; + buildDependencies = dependencyDerivations { + inherit features; + inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; + dependencies = crateConfig.buildDependencies or [ ]; + }; dependenciesWithRenames = let buildDeps = filterEnabledDependencies { @@ -844,45 +920,54 @@ rec { # } crateRenames = let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: + grouped = lib.groupBy (dependency: dependency.name) dependenciesWithRenames; + versionAndRename = + dep: let package = crateConfigs."${dep.packageId}"; in - { inherit (dep) rename; inherit (package) version; }; + { + inherit (dep) rename; + inherit (package) version; + }; in lib.mapAttrs (name: builtins.map versionAndRename) grouped; in - buildRustCrateForPkgsFunc pkgs - ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; - } - ); + buildRustCrateForPkgsFunc pkgs ( + crateConfig + // { + src = + crateConfig.src or (pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + }); + extraRustcOpts = + lib.lists.optional (targetFeatures != [ ]) + "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit + features + dependencies + buildDependencies + crateRenames + release + ; + } + ); in builtByPackageIdByPkgs; - /* Returns the actual derivations for the given dependencies. */ + # Returns the actual derivations for the given dependencies. dependencyDerivations = { buildByPackageId , features , dependencies , target + , }: assert (builtins.isList features); assert (builtins.isList dependencies); @@ -895,52 +980,59 @@ rec { in map depDerivation enabledDependencies; - /* Returns a sanitized version of val with all values substituted that cannot + /* + Returns a sanitized version of val with all values substituted that cannot be serialized as JSON. */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: sanitizeForJson) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; + sanitizeForJson = + val: + if builtins.isAttrs val then + lib.mapAttrs (n: sanitizeForJson) val + else if builtins.isList val then + builtins.map sanitizeForJson val + else if builtins.isFunction val then + "function" + else + val; - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; + # Returns various tools to debug a crate. + debugCrate = + { packageId + , target ? makeDefaultTarget stdenv.hostPlatform + , + }: + assert (builtins.isString packageId); + let + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson (buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; + }); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; }; + in + { + internal = debug; }; - in - { internal = debug; }; - /* Returns differences between cargo default features and crate2nix default + /* + Returns differences between cargo default features and crate2nix default features. This is useful for verifying the feature resolution in crate2nix. @@ -949,22 +1041,26 @@ rec { { crateConfigs ? crates , packageId , target + , }: assert (builtins.isAttrs crateConfigs); let prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + mergedFeatures = prefixValues "crate2nix" (mergePackageFeatures { + inherit crateConfigs packageId target; + features = [ "default" ]; + }); configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + combined = lib.foldAttrs (a: b: a // b) { } [ + mergedFeatures + configs + ]; + onlyInCargo = builtins.attrNames ( + lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined + ); + onlyInCrate2Nix = builtins.attrNames ( + lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined + ); differentFeatures = lib.filterAttrs ( n: v: @@ -978,7 +1074,8 @@ rec { inherit onlyInCargo onlyInCrate2Nix differentFeatures; }; - /* Returns an attrset mapping packageId to the list of enabled features. + /* + Returns an attrset mapping packageId to the list of enabled features. If multiple paths to a dependency enable different features, the corresponding feature sets are merged. Features in rust are additive. @@ -991,10 +1088,10 @@ rec { , dependencyPath ? [ crates.${packageId}.crateName ] , featuresByPackageId ? { } , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false + , # Adds devDependencies to the crate with rootPackageId. + runTests ? false , ... - } @ args: + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isString rootPackageId); @@ -1007,84 +1104,93 @@ rec { crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); expandedFeatures = expandFeatures (crateConfig.features or { }) features; enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: + depWithResolvedFeatures = + dependency: let inherit (dependency) packageId; features = dependencyFeatures enabledFeatures dependency; in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } + { + inherit packageId features; + }; + resolveDependencies = + cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache ( + cache: + { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then + cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit + crateConfigs + packageId + target + runTests + rootPackageId + ; + } ); cacheWithSelf = let cacheFeatures = featuresByPackageId.${packageId} or [ ]; combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in - featuresByPackageId // { + featuresByPackageId + // { "${packageId}" = combinedFeatures; }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + cacheWithDependencies = resolveDependencies cacheWithSelf "dep" ( + crateConfig.dependencies or [ ] + ++ lib.optionals (runTests && packageId == rootPackageId) (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = resolveDependencies cacheWithDependencies "build" ( + crateConfig.buildDependencies or [ ] + ); in cacheWithAll; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + # Returns the enabled dependencies given the enabled features. + filterEnabledDependencies = + { dependencies + , features + , target + , + }: + assert (builtins.isList dependencies); + assert (builtins.isList features); + assert (builtins.isAttrs target); - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) - ) - dependencies; + dependencies; - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: + # Returns whether the given feature should enable the given dependency. + doesFeatureEnableDependency = + dependency: feature: let name = dependency.rename or dependency.name; prefix = "${name}/"; @@ -1093,109 +1199,116 @@ rec { in feature == name || feature == "dep:" + name || startsWithPrefix; - /* Returns the expanded features for the given inputFeatures by applying the + /* + Returns the expanded features for the given inputFeatures by applying the rules in featureMap. featureMap is an attribute set which maps feature names to lists of further feature names to enable in case this feature is selected. */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; + expandFeatures = + featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; - /* This function adds optional dependencies as features if they are enabled + /* + This function adds optional dependencies as features if they are enabled indirectly by dependency features. This function mimics Cargo's behavior described in a note at: https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); + enableFeatures = + dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); + let + additionalFeatures = lib.concatMap + ( + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled then + [ (dependency.rename or dependency.name) ] + else + [ ] + ) + dependencies; + in + sortedUnique (features ++ additionalFeatures); /* Returns the actual features for the given dependency. features: The features of the crate that refers this dependency. */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; + dependencyFeatures = + features: dependency: + assert (builtins.isList features); + assert (builtins.isAttrs dependency); + let + defaultOrNil = if dependency.usesDefaultFeatures or true then [ "default" ] else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = + let + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: if lib.hasPrefix prefix s then lib.removePrefix prefix s else null; + extractFeature = + feature: + lib.findFirst (f: f != null) null ( + map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ] + ); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); + in + dependencyFeatures; + in + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + # Sorts and removes duplicates from a list of strings. + sortedUnique = + features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + deprecationWarning = + message: value: + if strictDeprecation then + builtins.throw "strictDeprecation enabled, aborting: ${message}" + else + builtins.trace message value; # # crate2nix/default.nix (excerpt end) diff --git a/sample_projects/sub_dir_crates/Cargo.nix b/sample_projects/sub_dir_crates/Cargo.nix index 6f2bdd02..5b16ffb6 100644 --- a/sample_projects/sub_dir_crates/Cargo.nix +++ b/sample_projects/sub_dir_crates/Cargo.nix @@ -5,6 +5,7 @@ { nixpkgs ? , pkgs ? import nixpkgs { config = {}; } +, fetchurl ? pkgs.fetchurl , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? pkgs: pkgs.buildRustCrate @@ -61,6 +62,8 @@ rec { }; }; + + # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { name = "all-workspace-members"; @@ -148,7 +151,8 @@ rec { # crate2nix/default.nix (excerpt start) # - /* Target (platform) data for conditional dependencies. + /* + Target (platform) data for conditional dependencies. This corresponds roughly to what buildRustCrate is setting. */ makeDefaultTarget = platform: { @@ -160,30 +164,70 @@ rec { inherit (platform.rust.platform) arch os - vendor; + vendor + ; family = platform.rust.platform.target-family; env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; + endian = if platform.parsed.cpu.significantByte.name == "littleEndian" then "little" else "big"; pointer_width = toString platform.parsed.cpu.bits; debug_assertions = false; }; - /* Filters common temp files and build files. */ + registryUrl = + { registries + , url + , crate + , version + , sha256 + , + }: + let + dl = registries.${url}.dl; + tmpl = [ + "{crate}" + "{version}" + "{prefix}" + "{lowerprefix}" + "{sha256-checksum}" + ]; + in + with lib.strings; + if lib.lists.any (i: hasInfix "{}" dl) tmpl then + let + prefix = + if builtins.stringLength crate == 1 then + "1" + else if builtins.stringLength crate == 2 then + "2" + else + "${builtins.substring 0 2 crate}/${builtins.substring 2 (builtins.stringLength crate - 2) crate}"; + in + builtins.replaceStrings tmpl [ + crate + version + prefix + (lib.strings.toLower prefix) + sha256 + ] + else + "${dl}/${crate}/${version}/download"; + + # Filters common temp files and build files. # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: + sourceFilter = + name: type: let baseName = builtins.baseNameOf (builtins.toString name); in - ! ( + !( # Filter out git baseName == ".gitignore" || (type == "directory" && baseName == ".git") # Filter out build results || ( - type == "directory" && ( + type == "directory" + && ( baseName == "target" || baseName == "_site" || baseName == ".sass-cache" @@ -193,16 +237,11 @@ rec { ) # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + || (type == "symlink" && lib.hasPrefix "result" baseName) # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" - ) - ) || lib.hasSuffix ".iml" baseName + || (type == "directory" && (baseName == ".idea" || baseName == ".vscode")) + || lib.hasSuffix ".iml" baseName # Filter out nix build files || baseName == "Cargo.nix" @@ -216,90 +255,100 @@ rec { || baseName == "tests.nix" ); - /* Returns a crate which depends on successful test execution + /* + Returns a crate which depends on successful test execution of crate given as the second argument. testCrateFlags: list of flags to pass to the test exectuable testInputs: list of packages that should be available during test execution */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - } + crateWithTest = + { crate + , testCrate + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + , + }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override (_: { + buildTests = true; + }); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" ( + pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ] ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.stdenvNoCC.mkDerivation { - name = "run-tests-${testCrate.name}"; - - inherit (crate) src; - - inherit testCrateFlags; - - buildInputs = testInputs; - - buildPhase = '' - set -e - export RUST_BACKTRACE=1 - - # build outputs - testRoot=target/debug - mkdir -p $testRoot - - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . - - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - }; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); - - /* A restricted overridable version of builtRustCratesWithFeatures. */ + in + pkgs.stdenvNoCC.mkDerivation { + name = "run-tests-${testCrate.name}"; + + inherit (crate) src; + + inherit testCrateFlags; + + buildInputs = testInputs; + + buildPhase = '' + set -e + export RUST_BACKTRACE=1 + + # build outputs + testRoot=target/debug + mkdir -p $testRoot + + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . + + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + }; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; + }; + } + ( + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + '' + ); + + # A restricted overridable version of builtRustCratesWithFeatures. buildRustCrateWithFeatures = { packageId , features ? rootFeatures @@ -308,10 +357,11 @@ rec { , runTests ? false , testCrateFlags ? [ ] , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" + , # Any command to run immediatelly before a test is executed. + testPreRun ? "" + , # Any command run immediatelly after a test is executed. + testPostRun ? "" + , }: lib.makeOverridable ( @@ -322,17 +372,19 @@ rec { , testInputs , testPreRun , testPostRun + , }: let buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc + if buildRustCrateForPkgsFunc != null then + buildRustCrateForPkgsFunc else ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs + if crateOverrides == pkgs.defaultCrateOverrides then + buildRustCrateForPkgs else - pkgs: (buildRustCrateForPkgs pkgs).override { + pkgs: + (buildRustCrateForPkgs pkgs).override { defaultCrateOverrides = crateOverrides; } ); @@ -354,15 +406,32 @@ rec { { crate = drv; testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; + inherit + testCrateFlags + testInputs + testPreRun + testPostRun + ; } - else drv; + else + drv; in derivation ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + { + inherit + features + crateOverrides + runTests + testCrateFlags + testInputs + testPreRun + testPostRun + ; + }; - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + /* + Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc for the corresponding crate. */ builtRustCratesWithFeatures = @@ -372,7 +441,8 @@ rec { , buildRustCrateForPkgsFunc , runTests , makeTarget ? makeDefaultTarget - } @ args: + , + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isList features); @@ -380,55 +450,61 @@ rec { assert (builtins.isBool runTests); let rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); + mergedFeatures = mergePackageFeatures ( + args + // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { + test = runTests; + }; + } + ); # Memoize built packages so that reappearing packages are only built once. builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + mkBuiltByPackageIdByPkgs = + pkgs: let self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; + crates = lib.mapAttrs + ( + packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId + ) + crateConfigs; target = makeTarget stdenv.hostPlatform; build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: + buildByPackageIdForPkgsImpl = + self: pkgs: packageId: let features = mergedFeatures."${packageId}" or [ ]; crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; - }; + crateConfig = builtins.removeAttrs crateConfig' [ + "resolvedDefaultFeatures" + "devDependencies" + ]; + devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( + crateConfig'.devDependencies or [ ] + ); + dependencies = dependencyDerivations { + inherit features; + inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; + dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; + }; + buildDependencies = dependencyDerivations { + inherit features; + inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; + dependencies = crateConfig.buildDependencies or [ ]; + }; dependenciesWithRenames = let buildDeps = filterEnabledDependencies { @@ -453,45 +529,54 @@ rec { # } crateRenames = let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: + grouped = lib.groupBy (dependency: dependency.name) dependenciesWithRenames; + versionAndRename = + dep: let package = crateConfigs."${dep.packageId}"; in - { inherit (dep) rename; inherit (package) version; }; + { + inherit (dep) rename; + inherit (package) version; + }; in lib.mapAttrs (name: builtins.map versionAndRename) grouped; in - buildRustCrateForPkgsFunc pkgs - ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; - } - ); + buildRustCrateForPkgsFunc pkgs ( + crateConfig + // { + src = + crateConfig.src or (pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + }); + extraRustcOpts = + lib.lists.optional (targetFeatures != [ ]) + "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit + features + dependencies + buildDependencies + crateRenames + release + ; + } + ); in builtByPackageIdByPkgs; - /* Returns the actual derivations for the given dependencies. */ + # Returns the actual derivations for the given dependencies. dependencyDerivations = { buildByPackageId , features , dependencies , target + , }: assert (builtins.isList features); assert (builtins.isList dependencies); @@ -504,52 +589,59 @@ rec { in map depDerivation enabledDependencies; - /* Returns a sanitized version of val with all values substituted that cannot + /* + Returns a sanitized version of val with all values substituted that cannot be serialized as JSON. */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: sanitizeForJson) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; + sanitizeForJson = + val: + if builtins.isAttrs val then + lib.mapAttrs (n: sanitizeForJson) val + else if builtins.isList val then + builtins.map sanitizeForJson val + else if builtins.isFunction val then + "function" + else + val; + + # Returns various tools to debug a crate. + debugCrate = + { packageId + , target ? makeDefaultTarget stdenv.hostPlatform + , + }: + assert (builtins.isString packageId); + let + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson (buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; + }); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; }; + in + { + internal = debug; }; - in - { internal = debug; }; - /* Returns differences between cargo default features and crate2nix default + /* + Returns differences between cargo default features and crate2nix default features. This is useful for verifying the feature resolution in crate2nix. @@ -558,22 +650,26 @@ rec { { crateConfigs ? crates , packageId , target + , }: assert (builtins.isAttrs crateConfigs); let prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + mergedFeatures = prefixValues "crate2nix" (mergePackageFeatures { + inherit crateConfigs packageId target; + features = [ "default" ]; + }); configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + combined = lib.foldAttrs (a: b: a // b) { } [ + mergedFeatures + configs + ]; + onlyInCargo = builtins.attrNames ( + lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined + ); + onlyInCrate2Nix = builtins.attrNames ( + lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined + ); differentFeatures = lib.filterAttrs ( n: v: @@ -587,7 +683,8 @@ rec { inherit onlyInCargo onlyInCrate2Nix differentFeatures; }; - /* Returns an attrset mapping packageId to the list of enabled features. + /* + Returns an attrset mapping packageId to the list of enabled features. If multiple paths to a dependency enable different features, the corresponding feature sets are merged. Features in rust are additive. @@ -600,10 +697,10 @@ rec { , dependencyPath ? [ crates.${packageId}.crateName ] , featuresByPackageId ? { } , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false + , # Adds devDependencies to the crate with rootPackageId. + runTests ? false , ... - } @ args: + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isString rootPackageId); @@ -616,84 +713,93 @@ rec { crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); expandedFeatures = expandFeatures (crateConfig.features or { }) features; enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: + depWithResolvedFeatures = + dependency: let inherit (dependency) packageId; features = dependencyFeatures enabledFeatures dependency; in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } + { + inherit packageId features; + }; + resolveDependencies = + cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache ( + cache: + { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then + cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit + crateConfigs + packageId + target + runTests + rootPackageId + ; + } ); cacheWithSelf = let cacheFeatures = featuresByPackageId.${packageId} or [ ]; combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in - featuresByPackageId // { + featuresByPackageId + // { "${packageId}" = combinedFeatures; }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + cacheWithDependencies = resolveDependencies cacheWithSelf "dep" ( + crateConfig.dependencies or [ ] + ++ lib.optionals (runTests && packageId == rootPackageId) (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = resolveDependencies cacheWithDependencies "build" ( + crateConfig.buildDependencies or [ ] + ); in cacheWithAll; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + # Returns the enabled dependencies given the enabled features. + filterEnabledDependencies = + { dependencies + , features + , target + , + }: + assert (builtins.isList dependencies); + assert (builtins.isList features); + assert (builtins.isAttrs target); - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) - ) - dependencies; + dependencies; - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: + # Returns whether the given feature should enable the given dependency. + doesFeatureEnableDependency = + dependency: feature: let name = dependency.rename or dependency.name; prefix = "${name}/"; @@ -702,109 +808,116 @@ rec { in feature == name || feature == "dep:" + name || startsWithPrefix; - /* Returns the expanded features for the given inputFeatures by applying the + /* + Returns the expanded features for the given inputFeatures by applying the rules in featureMap. featureMap is an attribute set which maps feature names to lists of further feature names to enable in case this feature is selected. */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; + expandFeatures = + featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; - /* This function adds optional dependencies as features if they are enabled + /* + This function adds optional dependencies as features if they are enabled indirectly by dependency features. This function mimics Cargo's behavior described in a note at: https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); + enableFeatures = + dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); + let + additionalFeatures = lib.concatMap + ( + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled then + [ (dependency.rename or dependency.name) ] + else + [ ] + ) + dependencies; + in + sortedUnique (features ++ additionalFeatures); /* Returns the actual features for the given dependency. features: The features of the crate that refers this dependency. */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + dependencyFeatures = + features: dependency: + assert (builtins.isList features); + assert (builtins.isAttrs dependency); + let + defaultOrNil = if dependency.usesDefaultFeatures or true then [ "default" ] else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = + let + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: if lib.hasPrefix prefix s then lib.removePrefix prefix s else null; + extractFeature = + feature: + lib.findFirst (f: f != null) null ( + map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ] + ); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); + in + dependencyFeatures; + in + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + # Sorts and removes duplicates from a list of strings. + sortedUnique = + features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; + + deprecationWarning = + message: value: + if strictDeprecation then + builtins.throw "strictDeprecation enabled, aborting: ${message}" + else + builtins.trace message value; # # crate2nix/default.nix (excerpt end)