Add fetchSubmodules to builtins.fetchGit

doc/manual/expressions/builtins.xml

@@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
             </para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term>fetchSubmodules</term>
+          <listitem>
+            <para>
+              A boolean parameter that specifies whether submodules
+              should be checked out. Defaults to
+              <literal>false</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
       </variablelist>
 
       <example>

src/libexpr/local.mk

@@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
 libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
+
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif

src/libexpr/primops/fetchGit.cc

@@ -22,14 +22,25 @@ struct GitInfo
     std::string rev;
     std::string shortRev;
     uint64_t revCount = 0;
+    bool submodules = false;
 };
 
 std::regex revRegex("^[0-9a-fA-F]{40}$");
 
+static bool isNotDotGitDirectory(const Path & path)
+{
+  static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
+
+  return not std::regex_match(path, gitDirRegex);
+}
+
 GitInfo exportGit(ref<Store> store, const std::string & uri,
     std::optional<std::string> ref, std::string rev,
-    const std::string & name)
+    const std::string & name, bool fetchSubmodules)
 {
+    GitInfo gitInfo;
+    gitInfo.submodules = fetchSubmodules;
+
     if (evalSettings.pureEval && rev == "")
         throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
 
@@ -47,12 +58,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (!clean) {
 
             /* This is an unclean working tree. So copy all tracked files. */
-            GitInfo gitInfo;
             gitInfo.rev = "0000000000000000000000000000000000000000";
             gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
+            auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" });
+            if (fetchSubmodules) {
+                gitOpts.emplace_back("--recurse-submodules");
+            }
             auto files = tokenizeString<std::set<std::string>>(
-                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
+                runProgram("git", true, gitOpts), "\0"s);
 
             PathFilter filter = [&](const Path & p) -> bool {
                 assert(hasPrefix(p, uri));
@@ -139,13 +153,13 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     }
 
     // FIXME: check whether rev is an ancestor of ref.
-    GitInfo gitInfo;
     gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
     gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
 
     printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
 
-    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
+    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev
+        + (fetchSubmodules ? "submodules" : "")).to_string(Base32, false);
     Path storeLink = cacheDir + "/" + storeLinkName + ".link";
     PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
 
@@ -165,18 +179,39 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (e.errNo != ENOENT) throw;
     }
 
-    auto source = sinkToSource([&](Sink & sink) {
-        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
-        gitOptions.standardOut = &sink;
-        runProgram2(gitOptions);
-    });
-
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
+    PathFilter filter = defaultPathFilter;
+
+    // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root
+    // repo is cached.
+    if (fetchSubmodules) {
+        Path tmpGitDir = createTempDir();
+        AutoDelete delTmpGitDir(tmpGitDir, true);
+
+        runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+        // TODO: the cacheDir repo might lack the ref (it only checks if rev
+        // exists, see FIXME above) so use a big hammer and fetch everything to
+        // ensure we get the rev.
+        runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
+                                  "--update-head-ok", "--", cacheDir, "refs/*:refs/*" });
+
+        runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev });
+        runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });
+        runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+
+	filter = isNotDotGitDirectory;
+    } else {
+        auto source = sinkToSource([&](Sink & sink) {
+            RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+            gitOptions.standardOut = &sink;
+            runProgram2(gitOptions);
+        });
 
-    unpackTarfile(*source, tmpDir);
+        unpackTarfile(*source, tmpDir);
+    }
 
-    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter));
 
     gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
 
@@ -186,6 +221,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
     json["name"] = name;
     json["rev"] = gitInfo.rev;
     json["revCount"] = gitInfo.revCount;
+    json["submodules"] = gitInfo.submodules;
 
     writeFile(storeLink, json.dump());
 
@@ -198,6 +234,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     std::optional<std::string> ref;
     std::string rev;
     std::string name = "source";
+    bool fetchSubmodules = false;
     PathSet context;
 
     state.forceValue(*args[0]);
@@ -216,6 +253,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
                 rev = state.forceStringNoCtx(*attr.value, *attr.pos);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else if (n == "submodules")
+                fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
             else
                 throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
         }
@@ -230,13 +269,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
     // whitelist. Ah well.
     state.checkURI(url);
 
-    auto gitInfo = exportGit(state.store, url, ref, rev, name);
+    auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules);
 
     state.mkAttrs(v, 8);
     mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
     mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
     mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
     mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
+    mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules);
     v.attrs->sort();
 
     if (state.allowedPaths)

tests/fetchGitSubmodules.sh

@@ -0,0 +1,97 @@
+source common.sh
+
+set -u
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git submodule tests"
+    exit 99
+fi
+
+clearStore
+
+rootRepo=$TEST_ROOT/gitSubmodulesRoot
+subRepo=$TEST_ROOT/gitSubmodulesSub
+
+rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2
+
+initGitRepo() {
+    git init $1
+    git -C $1 config user.email "foobar@example.com"
+    git -C $1 config user.name "Foobar"
+}
+
+addGitContent() {
+    echo "lorem ipsum" > $1/content
+    git -C $1 add content
+    git -C $1 commit -m "Initial commit"
+}
+
+initGitRepo $subRepo
+addGitContent $subRepo
+
+initGitRepo $rootRepo
+
+git -C $rootRepo submodule init
+git -C $rootRepo submodule add $subRepo sub
+git -C $rootRepo add sub
+git -C $rootRepo commit -m "Add submodule"
+
+rev=$(git -C $rootRepo rev-parse HEAD)
+
+r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
+r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r2 ]]
+[[ $r2 != $r3 ]]
+
+r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath")
+r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath")
+r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r4 ]]
+[[ $r4 == $r5 ]]
+[[ $r3 == $r6 ]]
+[[ $r6 == $r7 ]]
+[[ $r7 == $r8 ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
+[[ $have_submodules == true ]]
+
+pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
+
+# The resulting store path cannot be the same.
+[[ $pathWithoutSubmodules != $pathWithSubmodules ]]
+
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]
+
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]]
+
+# The submodules flag is actually honored.
+[[ ! -e $pathWithoutSubmodules/sub/content ]]
+[[ -e $pathWithSubmodules/sub/content ]]
+
+[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]]
+
+# No .git directory or submodule reference files must be left
+test "$(find "$pathWithSubmodules" -name .git)" = ""
+
+# Git repos without submodules can be fetched with submodules = true.
+subRev=$(git -C $subRepo rev-parse HEAD)
+noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
+noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
+
+[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]

tests/local.mk

@@ -17,6 +17,7 @@ nix_tests = \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
+  fetchGitSubmodules.sh \
   fetchMercurial.sh \
   signing.sh \
   run.sh \