Skip to content

Commit

Permalink
Make it able to inject non-remote file into RemoteInMemoryFileSystem.
Browse files Browse the repository at this point in the history
In following CLs, we want to implement 'copy' semantic for `ctx.actions.symlink` in RemoteInMemoryFileSystem, i.e. instead of creating a symlink in `#createSymbolicLink`, we want to copy the target metadata to the link path.

There are cases where the target metadata is not a remote metadata, e.g. symlink to source, symlink to output of local action, etc. So we need to support inject non-remote metadata into RemoteInMemoryFileSystem.

PiperOrigin-RevId: 536994360
Change-Id: I1696db007304d638903b7250b8478491aef76795
  • Loading branch information
coeuvre authored and copybara-github committed Jun 1, 2023
1 parent b7e3820 commit c997543
Showing 1 changed file with 21 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ void injectRemoteFile(PathFragment path, byte[] digest, long size, long expireAt
if (!isOutput(path)) {
return;
}
remoteOutputTree.injectRemoteFile(path, digest, size, expireAtEpochMilli);
var metadata =
RemoteFileArtifactValue.create(digest, size, /* locationIndex= */ 1, expireAtEpochMilli);
remoteOutputTree.injectFile(path, metadata);
}

void flush() throws IOException, InterruptedException {
Expand Down Expand Up @@ -574,11 +576,10 @@ private FileArtifactValue getMetadataByExecPath(PathFragment execPath) {
return m;
}

RemoteFileInfo remoteFile =
remoteOutputTree.getRemoteFileInfo(
execRoot.getRelative(execPath), /* followSymlinks= */ true);
if (remoteFile != null) {
return remoteFile.getMetadata();
var stat =
remoteOutputTree.statNullable(execRoot.getRelative(execPath), /* followSymlinks= */ true);
if (stat instanceof FileStatusWithMetadata) {
return ((FileStatusWithMetadata) stat).getMetadata();
}

return null;
Expand Down Expand Up @@ -783,44 +784,38 @@ protected synchronized OutputStream getOutputStream(PathFragment path, boolean a

@Override
protected FileInfo newFile(Clock clock, PathFragment path) {
return new RemoteFileInfo(clock);
return new RemoteInMemoryFileInfo(clock);
}

void injectRemoteFile(PathFragment path, byte[] digest, long size, long expireAtEpochMilli)
throws IOException {
protected void injectFile(PathFragment path, FileArtifactValue metadata) throws IOException {
createDirectoryAndParents(path.getParentDirectory());
InMemoryContentInfo node = getOrCreateWritableInode(path);
// If a node was already existed and is not a remote file node (i.e. directory or symlink node
// ), throw an error.
if (!(node instanceof RemoteFileInfo)) {
if (!(node instanceof RemoteInMemoryFileInfo)) {
throw new IOException("Could not inject into " + node);
}

RemoteFileInfo remoteFileInfo = (RemoteFileInfo) node;

var metadata =
RemoteFileArtifactValue.create(digest, size, /* locationIndex= */ 1, expireAtEpochMilli);
remoteFileInfo.set(metadata);
RemoteInMemoryFileInfo remoteInMemoryFileInfo = (RemoteInMemoryFileInfo) node;
remoteInMemoryFileInfo.set(metadata);
}

// Override for access within this class
@Nullable
RemoteFileInfo getRemoteFileInfo(PathFragment path, boolean followSymlinks) {
InMemoryContentInfo node = inodeStatErrno(path, followSymlinks).inode();
if (!(node instanceof RemoteFileInfo)) {
return null;
}
return (RemoteFileInfo) node;
@Override
protected FileStatus statNullable(PathFragment path, boolean followSymlinks) {
return super.statNullable(path, followSymlinks);
}
}

static class RemoteFileInfo extends FileInfo implements FileStatusWithMetadata {
private RemoteFileArtifactValue metadata;
static class RemoteInMemoryFileInfo extends FileInfo implements FileStatusWithMetadata {
private FileArtifactValue metadata;

RemoteFileInfo(Clock clock) {
RemoteInMemoryFileInfo(Clock clock) {
super(clock);
}

private void set(RemoteFileArtifactValue metadata) {
private void set(FileArtifactValue metadata) {
this.metadata = metadata;
}

Expand Down Expand Up @@ -854,12 +849,8 @@ public long getSize() {
return metadata.getSize();
}

public long getExpireAtEpochMilli() {
return metadata.getExpireAtEpochMilli();
}

@Override
public RemoteFileArtifactValue getMetadata() {
public FileArtifactValue getMetadata() {
return metadata;
}
}
Expand Down

0 comments on commit c997543

Please sign in to comment.