Skip to content

Commit

Permalink
feat: simplify git dependency detection by only using cargo lock, all…
Browse files Browse the repository at this point in the history 
…Refs support
  • Loading branch information
@yusdacra
yusdacra committed Mar 18, 2022
1 parent 2fc8ce9 commit bd4822e
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 101 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ it is converted to an attribute set equivalent to `{ root = theArg; }`.
| `version` | The version of the derivation. |
| `src` | Used by `naersk` as source input to the derivation. When `root` is not set, `src` is also used to discover the `Cargo.toml` and `Cargo.lock`. |
| `root` | Used by `naersk` to read the `Cargo.toml` and `Cargo.lock` files. May be different from `src`. When `src` is not set, `root` is (indirectly) used as `src`. |
| `allRefs` | Whether to fetch all refs while fetching Git dependencies. Useful if the wanted revision isn't in the default branch. |
| `cargoBuild` | The command to use for the build. The argument must be a function modifying the default value. <br/> Default: `''cargo $cargo_options build $cargo_build_options >> $cargo_build_output_json''` |
| `cargoBuildOptions` | Options passed to cargo build, i.e. `cargo build <OPTS>`. These options can be accessed during the build through the environment variable `cargo_build_options`. <br/> Note: naersk relies on the `--out-dir out` option and the `--message-format` option. The `$cargo_message_format` variable is set based on the cargo version.<br/> Note: these values are not (shell) escaped, meaning that you can use environment variables but must be careful when introducing e.g. spaces. <br/> The argument must be a function modifying the default value. <br/> Default: `[ "$cargo_release" ''-j "$NIX_BUILD_CORES"'' "--message-format=$cargo_message_format" ]` |
| `remapPathPrefix` | When `true`, rustc remaps the (`/nix/store`) source paths to `/sources` to reduce the number of dependencies in the closure. Default: `true` |
Expand Down
18 changes: 7 additions & 11 deletions build.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,6 @@ let
builtins // import ./builtins
{ inherit lib writeText remarshal runCommand; };

# All the git dependencies, as a list
gitDependenciesList =
lib.concatLists (lib.mapAttrsToList (_: ds: ds) gitDependencies);

# This unpacks all git dependencies:
# $out/rand
# $out/rand/Cargo.toml
Expand Down Expand Up @@ -129,7 +125,7 @@ let
fi
done <<< "$tomls"
done < <(cat ${
builtins.toFile "git-deps-json" (builtins.toJSON gitDependenciesList)
builtins.toFile "git-deps-json" (builtins.toJSON gitDependencies)
} | jq -cMr '.[]')
'';

Expand Down Expand Up @@ -162,20 +158,20 @@ let
(
e:
let
key = if e ? rev then "rev=${e.rev}" else
if e ? tag then "tag=${e.tag}" else
if e ? branch then "branch=${e.branch}" else
throw "No 'rev', 'tag' or 'branch' specified";
key = if e ? rev then "?rev=${e.rev}" else
if e ? tag then "?tag=${e.tag}" else
if e ? branch then "?branch=${e.branch}" else
"";
in
{
name = "${e.url}?${key}";
name = "${e.url}${key}";
value = lib.filterAttrs (n: _: n == "rev" || n == "tag" || n == "branch") e // {
git = e.url;
replace-with = "nix-sources";
};
}
)
gitDependenciesList
gitDependencies
);
};

Expand Down
6 changes: 5 additions & 1 deletion config.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ let
# used as `src`.
root = attrs0.root or null;

# Whether to fetch all refs while fetching Git dependencies. Useful if
# the wanted revision isn't in the default branch.
allRefs = attrs0.allRefs or false;

# The command to use for the build.
cargoBuild =
allowFun attrs0 "cargoBuild"
Expand Down Expand Up @@ -250,7 +254,7 @@ let
# Whether we skip pre-building the deps
isSingleStep = attrs.singleStep;

inherit (attrs) overrideMain;
inherit (attrs) overrideMain allRefs;

# The members we want to build
# (list of directory names)
Expand Down
2 changes: 1 addition & 1 deletion default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ let
let
config = mkConfig arg;
gitDependencies =
libb.findGitDependencies { inherit (config) cargotomls cargolock; };
libb.findGitDependencies { inherit (config) cargolock allRefs; };
cargoconfig =
if builtinz.pathExists (toString config.root + "/.cargo/config")
then builtins.readFile (config.root + "/.cargo/config")
Expand Down
135 changes: 47 additions & 88 deletions lib.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,99 +64,58 @@ rec
mkMetadataKey = name: version:
"checksum ${name} ${version} (registry+https://github.com/rust-lang/crates.io-index)";

# a record:
# { "." = # '.' is the directory of the cargotoml
# [
# {
# name = "rand";
# url = "https://github.com/...";
# checkout = "/nix/store/checkout"
# }
# ]
# Gets all git dependencies in Cargo.lock as a list.
# [
# {
# name = "rand";
# url = "https://github.com/...";
# checkout = "/nix/store/checkout"
# }
# ]
findGitDependencies =
{ cargotomls
, cargolock
}:
let
# This returns all the git dependencies of a particular Cargo.toml.
# tomlDependencies : Cargo.toml ->
# [
# {
# checkout = { rev = "abcd"; shortRev = "abcd"; outPath = ...; ... };
# key = "abcd";
# name = "rand";
# url = "https://github.com/...";
# }
# ...
# ]
#
# NOTE: this is terribly inefficient _and_ confusing:
# * the lockfile is read once for every cargo toml entry (n^2)
# * 'fromLockfile' is odd, instead of returning the first matched
# revision it'll return a pseudo lockfile entry
tomlDependencies = cargotoml:
lib.filter (x: ! isNull x) (
lib.mapAttrsToList
(entryName: v: # The dependecy name + the entry from the cargo toml
if ! (lib.isAttrs v && builtins.hasAttr "git" v)
then null
else
let
# Use the 'package' attribute if it exists, which means this is a renamed dependency
# https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml
depName = v.package or entryName;
# predicate that holds if the given lockfile entry is the
# Cargo.toml dependency being looked at (depName)
pred = entry: entry.name == depName && (lib.substring 0 (4 + lib.stringLength v.git) entry.source) == "git+${v.git}";

# extract the revision from a Cargo.lock "package.source"
# entry
# git+https://gi.../rand?rev=703452...#703452
# ^^^^^^
extractRevision = url: lib.last (lib.splitString "#" url);

# parse a lockfile entry:
# { name = "rand";
# version = ...;
# source = "git+https://...rev=703452...#703452";
# } ->
# { name = "rand";
# source = "git+https://...rev=703452...#703452";
# revision = "703452";
# }
parseLockfileEntry = entry: rec { inherit (entry) name source; revision = extractRevision source; };

# List of all entries: [ { name, source, revision } ]
packageLocks = builtins.map parseLockfileEntry (lib.filter pred cargolock.package);
{ cargolock, allRefs }:
let
query = p: (lib.substring 0 4 (p.source or "")) == "git+";

# Find the first revision from the lockfile that:
# * has the same name as the cargo toml entry _if_ the cargo toml does not specify a revision, or
# * has the same name and revision as the cargo toml entry
fromLockfile = lib.findFirst (p: p.name == depName && ((! v?rev) || v.rev == p.revision)) null packageLocks;
extractRevision = source: lib.last (lib.splitString "#" source);
extractPart = part: source: if lib.hasInfix part source then lib.last (lib.splitString part (lib.head (lib.splitString "#" source))) else null;
extractRepoUrl = source:
let
splitted = lib.head (lib.splitString "?" source);
split = lib.substring 4 (lib.stringLength splitted) splitted;
in lib.head (lib.splitString "#" split);

# Cargo.lock revision is prioritized, because in Cargo.toml short revisions are allowed
val = v // { rev = fromLockfile.revision or v.rev or null; };
in
lib.filterAttrs (n: _: n == "rev" || n == "tag" || n == "branch") val //
{
name = depName;
url = val.git;
key = val.rev or val.tag or val.branch or
(throw "No 'rev', 'tag' or 'branch' available to specify key, nor a git revision was found in Cargo.lock");
checkout = builtins.fetchGit ({
url = val.git;
} // lib.optionalAttrs (val ? rev) {
rev = val.rev;
} // lib.optionalAttrs (val ? branch) {
ref = val.branch;
} // lib.optionalAttrs (val ? tag) {
ref = "refs/tags/${val.tag}";
});
}
) cargotoml.dependencies or { });
parseLock = lock:
let
source = lock.source;
rev = extractPart "?rev=" source;
tag = extractPart "?tag=" source;
branch = extractPart "?branch=" source;
in
lib.mapAttrs (_: x: tomlDependencies x) cargotomls;
{
inherit (lock) name;
revision = extractRevision source;
url = extractRepoUrl source;
} // (lib.optionalAttrs (! isNull branch) { inherit branch; })
// (lib.optionalAttrs (! isNull tag) { inherit tag; })
// (lib.optionalAttrs (! isNull rev) { inherit rev; });
packageLocks = builtins.map parseLock (lib.filter query cargolock.package);

mkFetch = lock: {
key = lock.rev or lock.tag or lock.branch or lock.revision
or (throw "No 'rev', 'tag' or 'branch' available to specify key, nor a git revision was found in Cargo.lock");
checkout = builtins.fetchGit ({
url = lock.url;
rev = lock.revision;
} // lib.optionalAttrs (lock ? branch) {
ref = lock.branch;
} // lib.optionalAttrs (lock ? tag) {
ref = lock.tag;
} // lib.optionalAttrs allRefs {
allRefs = true;
});
} // lock;
in lib.foldl' (acc: e: if lib.any (oe: (oe.url == e.url) && (oe.key == e.key)) acc then acc else acc ++ [e]) [] (builtins.map mkFetch packageLocks);

# A very minimal 'src' which makes cargo happy nonetheless
dummySrc =
Expand Down

0 comments on commit bd4822e

Please sign in to comment.