Skip to content

Commit

Permalink
Pull maven -src jars when available alongside jars re:
Browse files Browse the repository at this point in the history 
bazelbuild#308

RELNOTES: None.
PiperOrigin-RevId: 166219871
  • Loading branch information
@juliexxia @damienmg
juliexxia authored and damienmg committed Aug 24, 2017
1 parent 49bb723 commit 7a7c41d
Show file tree
Hide file tree
Showing 6 changed files with 245 additions and 63 deletions.
88 changes: 70 additions & 18 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/JarDecompressor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package com.google.devtools.build.lib.bazel.repository;

import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
Expand All @@ -34,42 +35,54 @@ public class JarDecompressor implements Decompressor {
protected JarDecompressor() {
}

/**
* The .jar can be used compressed, so this just exposes it in a way Bazel can use.
*
* <p>It moves the jar from some-name/x/y/z/foo.jar to some-name/jar/foo.jar and creates a
* BUILD.bazel file containing one entry: the .jar.
*/
@Override
@Nullable
public Path decompress(DecompressorDescriptor descriptor) throws RepositoryFunctionException {
// Example: archiveFile is external/some-name/foo.jar.
String baseName = descriptor.archivePath().getBaseName();
return decompressWithSrcjar(descriptor, Optional.absent());
}

/**
* The jar (and srcjar if available) can be used compressed, so this just exposes them in a way
* Bazel can use.
*
* <p>It moves the jar(s) from some-name/x/y/z/foo.jar to some-name/jar/foo.jar and creates a
* BUILD.bazel file containing only the jar(s).
*/
public Path decompressWithSrcjar(
DecompressorDescriptor descriptor, Optional<DecompressorDescriptor> srcjarDescriptor)
throws RepositoryFunctionException {
try {
// external/some-name/
FileSystemUtils.createDirectoryAndParents(descriptor.repositoryPath());
// external/some-name/WORKSPACE.
// external/some-name/WORKSPACE
RepositoryFunction.createWorkspaceFile(
descriptor.repositoryPath(), descriptor.targetKind(), descriptor.targetName());
// external/some-name/jar.
// external/some-name/jar/
Path jarDirectory = descriptor.repositoryPath().getRelative(getPackageName());
FileSystemUtils.createDirectoryAndParents(jarDirectory);
// external/some-name/repository/jar/foo.jar is a symbolic link to the jar in
// external/some-name.
Path jarSymlink = jarDirectory.getRelative(baseName);
if (!jarSymlink.exists()) {
jarSymlink.createSymbolicLink(descriptor.archivePath());
}
// external/some-name/repository/jar/BUILD.bazel defines the //jar target.
// external/some-name/jar/BUILD.bazel defines the //jar target.
Path buildFile = jarDirectory.getRelative("BUILD.bazel");

// Example: get foo.jar from external/some-name/foo.jar.
String baseName = descriptor.archivePath().getBaseName();
makeSymlink(descriptor);
String buildFileContents;
if (srcjarDescriptor.isPresent()) {
String srcjarBaseName = srcjarDescriptor.get().archivePath().getBaseName();
makeSymlink(srcjarDescriptor.get());
buildFileContents = createBuildFileWithSrcjar(baseName, srcjarBaseName);
} else {
buildFileContents = createBuildFile(baseName);
}

FileSystemUtils.writeLinesAs(
buildFile,
Charset.forName("UTF-8"),
"# DO NOT EDIT: automatically generated BUILD.bazel file for "
+ descriptor.targetKind()
+ " rule "
+ descriptor.targetName(),
createBuildFile(baseName));
buildFileContents);
if (descriptor.executable()) {
descriptor.archivePath().chmod(0755);
}
Expand All @@ -80,6 +93,25 @@ public Path decompress(DecompressorDescriptor descriptor) throws RepositoryFunct
return descriptor.repositoryPath();
}

/*
* Links some-name/x/y/z/foo.jar to some-name/jar/foo.jar (represented by {@code descriptor}
*/
protected void makeSymlink(DecompressorDescriptor descriptor) throws RepositoryFunctionException {
try {
Path jarDirectory = descriptor.repositoryPath().getRelative(getPackageName());
// external/some-name/jar/foo.jar is a symbolic link to the jar in
// external/some-name.
Path jarSymlink = jarDirectory.getRelative(descriptor.archivePath().getBaseName());
if (!jarSymlink.exists()) {
jarSymlink.createSymbolicLink(descriptor.archivePath());
}
} catch (IOException e) {
throw new RepositoryFunctionException(
new IOException("Error auto-creating jar repo structure: " + e.getMessage()),
Transience.TRANSIENT);
}
}

protected String getPackageName() {
return "jar";
}
Expand All @@ -99,4 +131,24 @@ protected String createBuildFile(String baseName) {
" visibility = ['//visibility:public']",
")");
}

protected String createBuildFileWithSrcjar(String baseName, String srcjarBaseName) {
return Joiner.on("\n")
.join(
"java_import(",
" name = 'jar',",
" jars = ['" + baseName + "'],",
" srcjar = '" + srcjarBaseName + "',",
" visibility = ['//visibility:public']",
")",
"",
"filegroup(",
" name = 'file',",
" srcs = [",
" '" + baseName + "',",
" '" + srcjarBaseName + "',",
" ],",
" visibility = ['//visibility:public']",
")");
}
}
102 changes: 76 additions & 26 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package com.google.devtools.build.lib.bazel.repository;

import com.google.common.base.Optional;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -72,69 +73,84 @@ public Path getOutputDirectory() {
}

/**
* Download the Maven artifact to the output directory. Returns the path to the jar.
* Download the Maven artifact to the output directory. Returns the path to the jar (and the
* srcjar if available).
*/
public Path download(String name, WorkspaceAttributeMapper mapper, Path outputDirectory,
public JarPaths download(String name, WorkspaceAttributeMapper mapper, Path outputDirectory,
MavenServerValue serverValue) throws IOException, EvalException {
this.name = name;
this.outputDirectory = outputDirectory;

String url = serverValue.getUrl();
Server server = serverValue.getServer();

// Initialize maven artifacts
Artifact artifact;
String artifactId = mapper.get("artifact", Type.STRING);
String sha1 = mapper.isAttributeValueExplicitlySpecified("sha1")
? mapper.get("sha1", Type.STRING) : null;
if (sha1 != null && !KeyType.SHA1.isValid(sha1)) {
throw new IOException("Invalid SHA-1 for maven_jar " + name + ": '" + sha1 + "'");
}
String artifactCoords = mapper.get("artifact", Type.STRING);
String sha1 =
mapper.isAttributeValueExplicitlySpecified("sha1") ? mapper.get("sha1", Type.STRING) : null;
if (sha1 != null && !KeyType.SHA1.isValid(sha1)) {
throw new IOException("Invalid SHA-1 for maven_jar " + name + ": '" + sha1 + "'");
}

try {
artifact = new DefaultArtifact(artifactId);
artifact = new DefaultArtifact(artifactCoords);
} catch (IllegalArgumentException e) {
throw new IOException(e.getMessage());
}

boolean isCaching = repositoryCache.isEnabled() && KeyType.SHA1.isValid(sha1);

if (isCaching) {
Path downloadPath = getDownloadDestination(artifact);
Path cachedDestination = repositoryCache.get(sha1, downloadPath, KeyType.SHA1);
if (cachedDestination != null) {
return cachedDestination;
return new JarPaths(cachedDestination, Optional.absent());
}
}

// Setup env for fetching jars
MavenConnector connector = new MavenConnector(outputDirectory.getPathString());
RepositorySystem system = connector.newRepositorySystem();
RepositorySystemSession session = connector.newRepositorySystemSession(system);
RemoteRepository repository =
new RemoteRepository.Builder(name, MavenServerValue.DEFAULT_ID, url)
.setAuthentication(new MavenAuthentication(server))
.build();

RemoteRepository repository = new RemoteRepository.Builder(
name, MavenServerValue.DEFAULT_ID, url)
.setAuthentication(new MavenAuthentication(server))
.build();
ArtifactRequest artifactRequest = new ArtifactRequest();

artifactRequest.setArtifact(artifact);
artifactRequest.setRepositories(ImmutableList.of(repository));

// Try fetching jar.
final Path jarDownload;
try {
ArtifactResult artifactResult = system.resolveArtifact(session, artifactRequest);
artifact = artifactResult.getArtifact();
artifact = downloadArtifact(artifact, repository, session, system);
} catch (ArtifactResolutionException e) {
throw new IOException("Failed to fetch Maven dependency: " + e.getMessage());
}

Path downloadPath = outputDirectory.getRelative(artifact.getFile().getAbsolutePath());
// Try also fetching srcjar.
Artifact artifactWithSrcs = srcjarCoords(artifact);
try {
artifactWithSrcs = downloadArtifact(artifactWithSrcs, repository, session, system);
} catch (ArtifactResolutionException e) {
// Intentionally ignored - missing srcjar is not an error.
}

jarDownload = outputDirectory.getRelative(artifact.getFile().getAbsolutePath());
// Verify checksum.
if (!Strings.isNullOrEmpty(sha1)) {
RepositoryCache.assertFileChecksum(sha1, downloadPath, KeyType.SHA1);
RepositoryCache.assertFileChecksum(sha1, jarDownload, KeyType.SHA1);
}

if (isCaching) {
repositoryCache.put(sha1, downloadPath, KeyType.SHA1);
repositoryCache.put(sha1, jarDownload, KeyType.SHA1);
}

if (artifactWithSrcs.getFile() != null) {
Path srcjarDownload =
outputDirectory.getRelative(artifactWithSrcs.getFile().getAbsolutePath());
return new JarPaths(jarDownload, Optional.fromNullable(srcjarDownload));
} else {
return new JarPaths(jarDownload, Optional.absent());
}
return downloadPath;
}

private Path getDownloadDestination(Artifact artifact) {
Expand All @@ -149,6 +165,40 @@ private Path getDownloadDestination(Artifact artifact) {
return outputDirectory.getRelative(joiner.toString());
}

private Artifact srcjarCoords(Artifact jar) {
return new DefaultArtifact(
jar.getGroupId(), jar.getArtifactId(), "sources", jar.getExtension(), jar.getVersion());
}

/*
* Set up request for and resolve (retrieve to local repo) artifact
*/
private Artifact downloadArtifact(
Artifact artifact,
RemoteRepository repository,
RepositorySystemSession session,
RepositorySystem system)
throws ArtifactResolutionException {
ArtifactRequest artifactRequest = new ArtifactRequest();
artifactRequest.setArtifact(artifact);
artifactRequest.setRepositories(ImmutableList.of(repository));
ArtifactResult artifactResult = system.resolveArtifact(session, artifactRequest);
return artifactResult.getArtifact();
}

/*
* Class for packaging srcjar and jar paths together when srcjar is available.
*/
static class JarPaths {
final Path jar;
@Nullable final Optional<Path> srcjar;

private JarPaths(Path jar, Optional<Path> srcjar) {
this.jar = jar;
this.srcjar = srcjar;
}
}

private static class MavenAuthentication implements Authentication {

private final Map<String, String> authenticationInfo;
Expand Down
31 changes: 23 additions & 8 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@

package com.google.devtools.build.lib.bazel.repository;

import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.bazel.repository.MavenDownloader.JarPaths;
import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue;
Expand Down Expand Up @@ -98,7 +100,7 @@ public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory,
if (env.valuesMissing()) {
return null;
}

Path outputDir = getExternalRepositoryDirectory(directories).getRelative(rule.getName());
return createOutputTree(rule, outputDir, serverValue);
}
Expand All @@ -110,24 +112,37 @@ private RepositoryDirectoryValue.Builder createOutputTree(Rule rule, Path output

createDirectory(outputDirectory);
String name = rule.getName();
Path repositoryJar;
final JarPaths repositoryJars;
try {
repositoryJar = mavenDownloader.download(
name, WorkspaceAttributeMapper.of(rule), outputDirectory, serverValue);
repositoryJars =
mavenDownloader.download(
name, WorkspaceAttributeMapper.of(rule), outputDirectory, serverValue);
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
} catch (EvalException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}

// Add a WORKSPACE file & BUILD file to the Maven jar.
Path result = DecompressorValue.decompress(DecompressorDescriptor.builder()
DecompressorDescriptor jar = getDescriptorBuilder(name, repositoryJars.jar, outputDirectory);
DecompressorDescriptor srcjar =
repositoryJars.srcjar.isPresent()
? getDescriptorBuilder(name, repositoryJars.srcjar.get(), outputDirectory)
: null;
JarDecompressor decompressor = (JarDecompressor) jar.getDecompressor();
Path result = decompressor.decompressWithSrcjar(jar, Optional.fromNullable(srcjar));
return RepositoryDirectoryValue.builder().setPath(result);
}

private DecompressorDescriptor getDescriptorBuilder(String name, Path jar, Path outputDirectory)
throws RepositoryFunctionException, InterruptedException {
return DecompressorDescriptor.builder()
.setDecompressor(JarDecompressor.INSTANCE)
.setTargetKind(MavenJarRule.NAME)
.setTargetName(name)
.setArchivePath(repositoryJar)
.setRepositoryPath(outputDirectory).build());
return RepositoryDirectoryValue.builder().setPath(result);
.setArchivePath(jar)
.setRepositoryPath(outputDirectory)
.build();
}

/**
Expand Down
Loading

0 comments on commit 7a7c41d

Please sign in to comment.