Skip to content

Commit

Permalink
Find runfiles in directories that are themselves runfiles
Browse files Browse the repository at this point in the history
When a target has a runfile that is contained in a directory that is itself one of its runfiles, the runfile will be shadowed by the `SymlinkEntry` for the directory. While this still allows to resolve the file in the runfiles symlink tree, a manifest-based lookup will fail. Since the manifest is used to create the symlink tree for which it is important that no path is a prefix of another, this can only be fixed in the runfiles libraries.

This PR extends the lookup logic of the Bash, C++, Java, and Python runfiles libraries to also find runfiles contained within directories that are themselves runfiles. It does so by searching the manifest not only for the exact provided rlocation path, but also for all path prefixes. If a prefix is looked up successfully, the corresponding suffix is resolved relative to the looked up path.

Fixes bazelbuild#14336.

Closes bazelbuild#14335.

PiperOrigin-RevId: 426894517
  • Loading branch information
fmeum authored and copybara-github committed Feb 7, 2022
1 parent c23410c commit 486d153
Show file tree
Hide file tree
Showing 8 changed files with 145 additions and 36 deletions.
47 changes: 41 additions & 6 deletions tools/bash/runfiles/runfiles.bash
Original file line number Diff line number Diff line change
Expand Up @@ -122,16 +122,51 @@ function rlocation() {
echo >&2 "INFO[runfiles.bash]: rlocation($1): looking in RUNFILES_MANIFEST_FILE ($RUNFILES_MANIFEST_FILE)"
fi
local -r result=$(grep -m1 "^$1 " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-)
if [[ -e "${result:-/dev/null}" ]]; then
if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($result)"
fi
echo "$result"
else
if [[ -z "$result" ]]; then
# If path references a runfile that lies under a directory that itself
# is a runfile, then only the directory is listed in the manifest. Look
# up all prefixes of path in the manifest and append the relative path
# from the prefix if there is a match.
local prefix="$1"
local prefix_result=
local new_prefix=
while true; do
new_prefix="${prefix%/*}"
[[ "$new_prefix" == "$prefix" ]] && break
prefix="$new_prefix"
prefix_result=$(grep -m1 "^$prefix " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-)
[[ -z "$prefix_result" ]] && continue
local -r candidate="${prefix_result}${1#"${prefix}"}"
if [[ -e "$candidate" ]]; then
if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($candidate) via prefix ($prefix)"
fi
echo "$candidate"
return 0
fi
# At this point, the manifest lookup of prefix has been successful,
# but the file at the relative path given by the suffix does not
# exist. We do not continue the lookup with a shorter prefix for two
# reasons:
# 1. Manifests generated by Bazel never contain a path that is a
# prefix of another path.
# 2. Runfiles libraries for other languages do not check for file
# existence and would have returned the non-existent path. It seems
# better to return no path rather than a potentially different,
# non-empty path.
break
done
if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
echo >&2 "INFO[runfiles.bash]: rlocation($1): not found in manifest"
fi
echo ""
else
if [[ -e "$result" ]]; then
if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($result)"
fi
echo "$result"
fi
fi
else
if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
Expand Down
14 changes: 13 additions & 1 deletion tools/bash/runfiles/runfiles_test.bash
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,14 @@ function test_init_manifest_based_runfiles() {
a/b $tmpdir/c/d
e/f $tmpdir/g h
y $tmpdir/y
c/dir $tmpdir/dir
EOF
mkdir "${tmpdir}/c"
mkdir "${tmpdir}/y"
mkdir -p "${tmpdir}/dir/deeply/nested"
touch "${tmpdir}/c/d" "${tmpdir}/g h"
touch "${tmpdir}/dir/file"
touch "${tmpdir}/dir/deeply/nested/file"

export RUNFILES_DIR=
export RUNFILES_MANIFEST_FILE=$tmpdir/foo.runfiles_manifest
Expand All @@ -153,10 +157,18 @@ EOF
[[ "$(rlocation a/b)" == "$tmpdir/c/d" ]] || fail
[[ "$(rlocation e/f)" == "$tmpdir/g h" ]] || fail
[[ "$(rlocation y)" == "$tmpdir/y" ]] || fail
rm -r "$tmpdir/c/d" "$tmpdir/g h" "$tmpdir/y"
[[ "$(rlocation c)" == "" ]] || fail
[[ "$(rlocation c/di)" == "" ]] || fail
[[ "$(rlocation c/dir)" == "$tmpdir/dir" ]] || fail
[[ "$(rlocation c/dir/file)" == "$tmpdir/dir/file" ]] || fail
[[ "$(rlocation c/dir/deeply/nested/file)" == "$tmpdir/dir/deeply/nested/file" ]] || fail
rm -r "$tmpdir/c/d" "$tmpdir/g h" "$tmpdir/y" "$tmpdir/dir"
[[ -z "$(rlocation a/b)" ]] || fail
[[ -z "$(rlocation e/f)" ]] || fail
[[ -z "$(rlocation y)" ]] || fail
[[ -z "$(rlocation c/dir)" ]] || fail
[[ -z "$(rlocation c/dir/file)" ]] || fail
[[ -z "$(rlocation c/dir/deeply/nested/file)" ]] || fail
}

function test_manifest_based_envvars() {
Expand Down
22 changes: 19 additions & 3 deletions tools/cpp/runfiles/runfiles_src.cc
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <unistd.h>
#endif // _WIN32

#include <algorithm>
#include <fstream>
#include <functional>
#include <map>
Expand Down Expand Up @@ -176,9 +177,24 @@ string Runfiles::Rlocation(const string& path) const {
if (IsAbsolute(path)) {
return path;
}
const auto value = runfiles_map_.find(path);
if (value != runfiles_map_.end()) {
return value->second;
const auto exact_match = runfiles_map_.find(path);
if (exact_match != runfiles_map_.end()) {
return exact_match->second;
}
if (!runfiles_map_.empty()) {
// If path references a runfile that lies under a directory that itself is a
// runfile, then only the directory is listed in the manifest. Look up all
// prefixes of path in the manifest and append the relative path from the
// prefix to the looked up path.
std::size_t prefix_end = path.size();
while ((prefix_end = path.find_last_of('/', prefix_end - 1)) !=
string::npos) {
const string prefix = path.substr(0, prefix_end);
const auto prefix_match = runfiles_map_.find(prefix);
if (prefix_match != runfiles_map_.end()) {
return prefix_match->second + "/" + path.substr(prefix_end + 1);
}
}
}
if (!directory_.empty()) {
return directory_ + "/" + path;
Expand Down
4 changes: 4 additions & 0 deletions tools/cpp/runfiles/runfiles_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,8 @@ TEST_F(RunfilesTest, ManifestBasedRunfilesRlocationAndEnvVars) {
EXPECT_EQ(r->Rlocation("/Foo"), "/Foo");
EXPECT_EQ(r->Rlocation("c:/Foo"), "c:/Foo");
EXPECT_EQ(r->Rlocation("c:\\Foo"), "c:\\Foo");
EXPECT_EQ(r->Rlocation("a/b/file"), "c/d/file");
EXPECT_EQ(r->Rlocation("a/b/deeply/nested/file"), "c/d/deeply/nested/file");
}

TEST_F(RunfilesTest, DirectoryBasedRunfilesRlocationAndEnvVars) {
Expand Down Expand Up @@ -316,6 +318,8 @@ TEST_F(RunfilesTest, ManifestAndDirectoryBasedRunfilesRlocationAndEnvVars) {
EXPECT_EQ(r->Rlocation("/Foo"), "/Foo");
EXPECT_EQ(r->Rlocation("c:/Foo"), "c:/Foo");
EXPECT_EQ(r->Rlocation("c:\\Foo"), "c:\\Foo");
EXPECT_EQ(r->Rlocation("a/b/file"), "c/d/file");
EXPECT_EQ(r->Rlocation("a/b/deeply/nested/file"), "c/d/deeply/nested/file");
AssertEnvvars(*r, mf->Path(), dir);
}

Expand Down
16 changes: 15 additions & 1 deletion tools/java/runfiles/Runfiles.java
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,21 @@ private static String findRunfilesDir(String manifest) {

@Override
public String rlocationChecked(String path) {
return runfiles.get(path);
String exactMatch = runfiles.get(path);
if (exactMatch != null) {
return exactMatch;
}
// If path references a runfile that lies under a directory that itself is a runfile, then
// only the directory is listed in the manifest. Look up all prefixes of path in the manifest
// and append the relative path from the prefix if there is a match.
int prefixEnd = path.length();
while ((prefixEnd = path.lastIndexOf('/', prefixEnd - 1)) != -1) {
String prefixMatch = runfiles.get(path.substring(0, prefixEnd));
if (prefixMatch != null) {
return prefixMatch + '/' + path.substring(prefixEnd + 1);
}
}
return null;
}

@Override
Expand Down
7 changes: 6 additions & 1 deletion tools/java/runfiles/testing/RunfilesTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -249,10 +249,15 @@ public void testManifestBasedRlocation() throws Exception {
new MockFile(
ImmutableList.of(
"Foo/runfile1 C:/Actual Path\\runfile1",
"Foo/Bar/runfile2 D:\\the path\\run file 2.txt"))) {
"Foo/Bar/runfile2 D:\\the path\\run file 2.txt",
"Foo/Bar/Dir E:\\Actual Path\\Directory"))) {
Runfiles r = Runfiles.createManifestBasedForTesting(mf.path.toString());
assertThat(r.rlocation("Foo/runfile1")).isEqualTo("C:/Actual Path\\runfile1");
assertThat(r.rlocation("Foo/Bar/runfile2")).isEqualTo("D:\\the path\\run file 2.txt");
assertThat(r.rlocation("Foo/Bar/Dir")).isEqualTo("E:\\Actual Path\\Directory");
assertThat(r.rlocation("Foo/Bar/Dir/File")).isEqualTo("E:\\Actual Path\\Directory/File");
assertThat(r.rlocation("Foo/Bar/Dir/Deeply/Nested/File"))
.isEqualTo("E:\\Actual Path\\Directory/Deeply/Nested/File");
assertThat(r.rlocation("unknown")).isNull();
}
}
Expand Down
17 changes: 16 additions & 1 deletion tools/python/runfiles/runfiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,22 @@ def __init__(self, path):
self._runfiles = _ManifestBased._LoadRunfiles(path)

def RlocationChecked(self, path):
return self._runfiles.get(path)
"""Returns the runtime path of a runfile."""
exact_match = self._runfiles.get(path)
if exact_match:
return exact_match
# If path references a runfile that lies under a directory that itself is a
# runfile, then only the directory is listed in the manifest. Look up all
# prefixes of path in the manifest and append the relative path from the
# prefix to the looked up path.
prefix_end = len(path)
while True:
prefix_end = path.rfind("/", 0, prefix_end - 1)
if prefix_end == -1:
return None
prefix_match = self._runfiles.get(path[0:prefix_end])
if prefix_match:
return prefix_match + "/" + path[prefix_end + 1:]

@staticmethod
def _LoadRunfiles(path):
Expand Down
54 changes: 31 additions & 23 deletions tools/python/runfiles/runfiles_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,26 +28,26 @@ def testRlocationArgumentValidation(self):
self.assertRaises(ValueError, lambda: r.Rlocation(None))
self.assertRaises(ValueError, lambda: r.Rlocation(""))
self.assertRaises(TypeError, lambda: r.Rlocation(1))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("../foo"))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("foo/.."))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("foo/../bar"))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("./foo"))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("foo/."))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("foo/./bar"))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("//foobar"))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("foo//"))
self.assertRaisesRegexp(ValueError, "is not normalized",
lambda: r.Rlocation("foo//bar"))
self.assertRaisesRegexp(ValueError, "is absolute without a drive letter",
lambda: r.Rlocation("\\foo"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("../foo"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("foo/.."))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("foo/../bar"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("./foo"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("foo/."))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("foo/./bar"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("//foobar"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("foo//"))
self.assertRaisesRegex(ValueError, "is not normalized",
lambda: r.Rlocation("foo//bar"))
self.assertRaisesRegex(ValueError, "is absolute without a drive letter",
lambda: r.Rlocation("\\foo"))

def testCreatesManifestBasedRunfiles(self):
with _MockFile(contents=["a/b c/d"]) as mf:
Expand Down Expand Up @@ -122,7 +122,7 @@ def testFailsToCreateManifestBasedBecauseManifestDoesNotExist(self):
def _Run():
runfiles.Create({"RUNFILES_MANIFEST_FILE": "non-existing path"})

self.assertRaisesRegexp(IOError, "non-existing path", _Run)
self.assertRaisesRegex(IOError, "non-existing path", _Run)

def testFailsToCreateAnyRunfilesBecauseEnvvarsAreNotDefined(self):
with _MockFile(contents=["a b"]) as mf:
Expand All @@ -140,14 +140,22 @@ def testFailsToCreateAnyRunfilesBecauseEnvvarsAreNotDefined(self):

def testManifestBasedRlocation(self):
with _MockFile(contents=[
"Foo/runfile1", "Foo/runfile2 C:/Actual Path\\runfile2",
"Foo/Bar/runfile3 D:\\the path\\run file 3.txt"
"Foo/runfile1",
"Foo/runfile2 C:/Actual Path\\runfile2",
"Foo/Bar/runfile3 D:\\the path\\run file 3.txt",
"Foo/Bar/Dir E:\\Actual Path\\Directory",
]) as mf:
r = runfiles.CreateManifestBased(mf.Path())
self.assertEqual(r.Rlocation("Foo/runfile1"), "Foo/runfile1")
self.assertEqual(r.Rlocation("Foo/runfile2"), "C:/Actual Path\\runfile2")
self.assertEqual(
r.Rlocation("Foo/Bar/runfile3"), "D:\\the path\\run file 3.txt")
self.assertEqual(
r.Rlocation("Foo/Bar/Dir/runfile4"),
"E:\\Actual Path\\Directory/runfile4")
self.assertEqual(
r.Rlocation("Foo/Bar/Dir/Deeply/Nested/runfile4"),
"E:\\Actual Path\\Directory/Deeply/Nested/runfile4")
self.assertIsNone(r.Rlocation("unknown"))
if RunfilesTest.IsWindows():
self.assertEqual(r.Rlocation("c:/foo"), "c:/foo")
Expand Down

0 comments on commit 486d153

Please sign in to comment.