Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
…4182)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
  • Loading branch information
reta authored Jun 13, 2024
1 parent 18c5bb6 commit 8af4647
Show file tree
Hide file tree
Showing 32 changed files with 467 additions and 77 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,28 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java
}

private JvmInstallationMetadata getJavaInstallation(File javaHome) {
final InstallationLocation location = new InstallationLocation(javaHome, "Java home");
InstallationLocation location = null;

try {
try {
// The InstallationLocation(File, String) is used by Gradle pre-8.8
location = (InstallationLocation) MethodHandles.publicLookup()
.findConstructor(InstallationLocation.class, MethodType.methodType(void.class, File.class, String.class))
.invokeExact(javaHome, "Java home");
} catch (Throwable ex) {
// The InstallationLocation::userDefined is used by Gradle post-8.7
location = (InstallationLocation) MethodHandles.publicLookup()
.findStatic(
InstallationLocation.class,
"userDefined",
MethodType.methodType(InstallationLocation.class, File.class, String.class)
)
.invokeExact(javaHome, "Java home");

}
} catch (Throwable ex) {
throw new IllegalStateException("Unable to find suitable InstallationLocation constructor / factory method", ex);
}

try {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,8 @@ private void configureGeneralTaskDefaults(Project project) {
project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> {
t.dependsOn(project.getTasks().withType(EmptyDirTask.class));
t.setIncludeEmptyDirs(true);
t.setDirMode(0755);
t.setFileMode(0644);
t.dirPermissions(perms -> perms.unix(0755));
t.filePermissions(perms -> perms.unix(0644));
});

// common config across all archives
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,14 @@
package org.opensearch.gradle.precommit;

import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin;
import org.opensearch.gradle.util.GradleUtils;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.TaskProvider;

public class DependencyLicensesPrecommitPlugin extends PrecommitPlugin {
Expand All @@ -48,15 +51,16 @@ public TaskProvider<? extends Task> createTask(Project project) {
TaskProvider<DependencyLicensesTask> dependencyLicenses = project.getTasks()
.register("dependencyLicenses", DependencyLicensesTask.class);

final Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
final Configuration compileOnly = project.getConfigurations()
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
final Provider<FileCollection> provider = project.provider(
() -> GradleUtils.getFiles(project, runtimeClasspath, dependency -> dependency instanceof ProjectDependency == false)
.minus(compileOnly)
);

// only require dependency licenses for non-opensearch deps
dependencyLicenses.configure(t -> {
Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
Configuration compileOnly = project.getConfigurations()
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
t.setDependencies(
runtimeClasspath.fileCollection(dependency -> dependency instanceof ProjectDependency == false).minus(compileOnly)
);
});
dependencyLicenses.configure(t -> t.getDependencies().set(provider));

// we also create the updateShas helper task that is associated with dependencyLicenses
project.getTasks().register("updateShas", UpdateShasTask.class, t -> t.setParentTask(dependencyLicenses));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.gradle.api.file.FileCollection;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.InputFiles;
Expand Down Expand Up @@ -121,7 +122,7 @@ public class DependencyLicensesTask extends DefaultTask {
/**
* A collection of jar files that should be checked.
*/
private FileCollection dependencies;
private Property<FileCollection> dependenciesProvider;

/**
* The directory to find the license and sha files in.
Expand Down Expand Up @@ -158,12 +159,11 @@ public void mapping(Map<String, String> props) {
}

@InputFiles
public FileCollection getDependencies() {
return dependencies;
}

public void setDependencies(FileCollection dependencies) {
this.dependencies = dependencies;
public Property<FileCollection> getDependencies() {
if (dependenciesProvider == null) {
dependenciesProvider = getProject().getObjects().property(FileCollection.class);
}
return dependenciesProvider;
}

@Optional
Expand All @@ -190,6 +190,11 @@ public void ignoreSha(String dep) {

@TaskAction
public void checkDependencies() throws IOException, NoSuchAlgorithmException {
if (dependenciesProvider == null) {
throw new GradleException("No dependencies variable defined.");
}

final FileCollection dependencies = dependenciesProvider.get();
if (dependencies == null) {
throw new GradleException("No dependencies variable defined.");
}
Expand Down Expand Up @@ -226,7 +231,7 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
}
}

checkDependencies(licenses, notices, sources, shaFiles);
checkDependencies(dependencies, licenses, notices, sources, shaFiles);

licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license"));

Expand Down Expand Up @@ -255,6 +260,7 @@ private void failIfAnyMissing(String item, Boolean exists, String type) {
}

private void checkDependencies(
FileCollection dependencies,
Map<String, Boolean> licenses,
Map<String, Boolean> notices,
Map<String, Boolean> sources,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.opensearch.gradle.LoggedExec;
import org.opensearch.gradle.OS;
import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin;
import org.opensearch.gradle.util.GradleUtils;
import org.gradle.api.DefaultTask;
import org.gradle.api.JavaVersion;
import org.gradle.api.artifacts.Configuration;
Expand Down Expand Up @@ -203,11 +204,13 @@ public Set<File> getJarsToScan() {
// or dependencies added as `files(...)`, we can't be sure if those are third party or not.
// err on the side of scanning these to make sure we don't miss anything
Spec<Dependency> reallyThirdParty = dep -> dep.getGroup() != null && dep.getGroup().startsWith("org.opensearch") == false;
Set<File> jars = getRuntimeConfiguration().getResolvedConfiguration().getFiles(reallyThirdParty);
Set<File> compileOnlyConfiguration = getProject().getConfigurations()
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
.getResolvedConfiguration()
.getFiles(reallyThirdParty);

Set<File> jars = GradleUtils.getFiles(getProject(), getRuntimeConfiguration(), reallyThirdParty).getFiles();
Set<File> compileOnlyConfiguration = GradleUtils.getFiles(
getProject(),
getProject().getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME),
reallyThirdParty
).getFiles();
// don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin
if (compileOnlyConfiguration != null) {
jars.removeAll(compileOnlyConfiguration);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public UpdateShasTask() {
public void updateShas() throws NoSuchAlgorithmException, IOException {
Set<File> shaFiles = parentTask.get().getShaFiles();

for (File dependency : parentTask.get().getDependencies()) {
for (File dependency : parentTask.get().getDependencies().get()) {
String jarName = dependency.getName();
File shaFile = parentTask.get().getShaFile(jarName);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
visitedSymbolicLinks.add(details.getFile());
final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString(), TarConstants.LF_SYMLINK);
entry.setModTime(getModTime(details));
entry.setMode(UnixStat.LINK_FLAG | details.getMode());
entry.setMode(UnixStat.LINK_FLAG | details.getPermissions().toUnixNumeric());
try {
entry.setLinkName(Files.readSymbolicLink(details.getFile().toPath()).toString());
tar.putArchiveEntry(entry);
Expand All @@ -197,7 +197,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
private void visitDirectory(final FileCopyDetailsInternal details) {
final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString() + "/");
entry.setModTime(getModTime(details));
entry.setMode(UnixStat.DIR_FLAG | details.getMode());
entry.setMode(UnixStat.DIR_FLAG | details.getPermissions().toUnixNumeric());
try {
tar.putArchiveEntry(entry);
tar.closeArchiveEntry();
Expand All @@ -209,7 +209,7 @@ private void visitDirectory(final FileCopyDetailsInternal details) {
private void visitFile(final FileCopyDetailsInternal details) {
final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString());
entry.setModTime(getModTime(details));
entry.setMode(UnixStat.FILE_FLAG | details.getMode());
entry.setMode(UnixStat.FILE_FLAG | details.getPermissions().toUnixNumeric());
entry.setSize(details.getSize());
try {
tar.putArchiveEntry(entry);
Expand Down
26 changes: 26 additions & 0 deletions buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,17 @@
import org.gradle.api.UnknownTaskException;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.artifacts.LenientConfiguration;
import org.gradle.api.file.FileCollection;
import org.gradle.api.internal.artifacts.ivyservice.ResolvedFilesCollectingVisitor;
import org.gradle.api.internal.artifacts.ivyservice.resolveengine.artifact.SelectedArtifactSet;
import org.gradle.api.plugins.JavaBasePlugin;
import org.gradle.api.plugins.JavaPluginExtension;
import org.gradle.api.provider.Provider;
import org.gradle.api.services.BuildService;
import org.gradle.api.services.BuildServiceRegistration;
import org.gradle.api.services.BuildServiceRegistry;
import org.gradle.api.specs.Spec;
import org.gradle.api.tasks.SourceSet;
import org.gradle.api.tasks.SourceSetContainer;
import org.gradle.api.tasks.TaskContainer;
Expand All @@ -53,6 +58,9 @@
import org.gradle.plugins.ide.eclipse.model.EclipseModel;
import org.gradle.plugins.ide.idea.model.IdeaModel;

import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -245,4 +253,22 @@ public static String getProjectPathFromTask(String taskPath) {
int lastDelimiterIndex = taskPath.lastIndexOf(":");
return lastDelimiterIndex == 0 ? ":" : taskPath.substring(0, lastDelimiterIndex);
}

public static FileCollection getFiles(Project project, Configuration cfg, Spec<Dependency> spec) {
final LenientConfiguration configuration = cfg.getResolvedConfiguration().getLenientConfiguration();
try {
// Using reflection here to cover the pre 8.7 releases (since those have no such APIs), the
// ResolverResults.LegacyResolverResults.LegacyVisitedArtifactSet::select(...) is not available
// on older versions.
final MethodHandle mh = MethodHandles.lookup()
.findVirtual(configuration.getClass(), "select", MethodType.methodType(SelectedArtifactSet.class, Spec.class))
.bindTo(configuration);

final ResolvedFilesCollectingVisitor visitor = new ResolvedFilesCollectingVisitor();
((SelectedArtifactSet) mh.invoke(spec)).visitArtifacts(visitor, false);
return project.files(visitor.getFiles());
} catch (Throwable ex) {
return project.files(configuration.getFiles(spec));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ private TaskProvider<DependencyLicensesTask> createDependencyLicensesTask(Projec
.register("dependencyLicenses", DependencyLicensesTask.class, new Action<DependencyLicensesTask>() {
@Override
public void execute(DependencyLicensesTask dependencyLicensesTask) {
dependencyLicensesTask.setDependencies(getDependencies(project));
dependencyLicensesTask.getDependencies().set(getDependencies(project));

final Map<String, String> mappings = new HashMap<>();
mappings.put("from", "groovy-.*");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile() throws IOE
public void whenDependencyAndWrongShaExistsThenShouldNotOverwriteShaFile() throws IOException, NoSuchAlgorithmException {
project.getDependencies().add("someCompileConfiguration", dependency);

File groovyJar = task.getParentTask().getDependencies().getFiles().iterator().next();
File groovyJar = task.getParentTask().getDependencies().get().getFiles().iterator().next();
String groovyShaName = groovyJar.getName() + ".sha1";

File groovySha = createFileIn(getLicensesDir(project), groovyShaName, "content");
Expand Down Expand Up @@ -162,7 +162,7 @@ private TaskProvider<DependencyLicensesTask> createDependencyLicensesTask(Projec
.register("dependencyLicenses", DependencyLicensesTask.class, new Action<DependencyLicensesTask>() {
@Override
public void execute(DependencyLicensesTask dependencyLicensesTask) {
dependencyLicensesTask.setDependencies(getDependencies(project));
dependencyLicensesTask.getDependencies().set(getDependencies(project));
}
});

Expand Down
20 changes: 15 additions & 5 deletions distribution/archives/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,17 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
with libFiles()
}
into('config') {
dirMode 0750
fileMode 0660
dirPermissions {
unix 0750
}
filePermissions {
unix 0660
}
with configFiles(distributionType, java)
from {
dirMode 0750
dirPermissions {
unix 0750
}
jvmOptionsDir.getParent()
}
}
Expand All @@ -61,13 +67,17 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
}
into('') {
from {
dirMode 0755
dirPermissions {
unix 0755
}
logsDir.getParent()
}
}
into('') {
from {
dirMode 0755
dirPermissions {
unix 0755
}
pluginsDir.getParent()
}
}
Expand Down
12 changes: 6 additions & 6 deletions distribution/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -363,9 +363,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
if (it.relativePath.segments[-2] == 'bin' || ((platform == 'darwin-x64' || platform == 'darwin-arm64') && it.relativePath.segments[-2] == 'MacOS')) {
// bin files, wherever they are within modules (eg platform specific) should be executable
// and MacOS is an alternative to bin on macOS
it.mode = 0755
it.permissions(perm -> perm.unix(0755))
} else {
it.mode = 0644
it.permissions(perm -> perm.unix(0644))
}
}
def buildModules = buildModulesTaskProvider
Expand Down Expand Up @@ -413,7 +413,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
from '../src/bin'
exclude '*.exe'
exclude '*.bat'
eachFile { it.setMode(0755) }
eachFile { it.permissions(perm -> perm.unix(0755)) }
MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java))
}
// windows files, only for zip
Expand All @@ -431,7 +431,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
// module provided bin files
with copySpec {
eachFile { it.setMode(0755) }
eachFile { it.permissions(perm -> perm.unix(0755)) }
from project(':distribution').buildBin
if (distributionType != 'zip') {
exclude '*.bat'
Expand Down Expand Up @@ -473,7 +473,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
eachFile { FileCopyDetails details ->
if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
details.mode = 0755
details.permissions(perm -> perm.unix(0755))
}
if (details.name == 'src.zip') {
details.exclude()
Expand Down Expand Up @@ -501,7 +501,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
eachFile { FileCopyDetails details ->
if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
details.mode = 0755
details.permissions(perm -> perm.unix(0755))
}
}
}
Expand Down
Loading

0 comments on commit 8af4647

Please sign in to comment.