From c777f40ff19698152ddebf8b714d150afb6d72ef Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Mon, 8 Jun 2020 23:02:31 +0900 Subject: [PATCH] * Add new `Builder.configDirectory` option for `Generator` to output files that GraalVM needs for AOT compilation (issue eclipse/deeplearning4j#7362) --- CHANGELOG.md | 1 + pom.xml | 4 + .../org/bytedeco/javacpp/tools/BuildMojo.java | 6 ++ .../org/bytedeco/javacpp/tools/Builder.java | 80 ++++++++++++++++++- .../org/bytedeco/javacpp/tools/Generator.java | 70 +++++++++++++++- 5 files changed, 153 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1478d0de9..5d01cc5ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + * Add new `Builder.configDirectory` option for `Generator` to output files that GraalVM needs for AOT compilation ([issue eclipse/deeplearning4j#7362](https://github.com/eclipse/deeplearning4j/issues/7362)) * Fix `Parser` error on `template<>` containing non-type parameters without names ([issue bytedeco/javacpp-presets#889](https://github.com/bytedeco/javacpp-presets/issues/889)) * Bundle also the `vcruntime140_1.dll` and `msvcp140_1.dll` redist files from Visual Studio * Fix `Builder` for different "java.home" path returned by latest JDKs from Oracle ([pull #400](https://github.com/bytedeco/javacpp/pull/400)) diff --git a/pom.xml b/pom.xml index de2d58d2c..f72b37655 100644 --- a/pom.xml +++ b/pom.xml @@ -192,6 +192,7 @@ ${javacpp.platform.library.path}/ ${javacpp.platform}${javacpp.platform.extension}/ org/bytedeco/javacpp/${javacpp.platform}${javacpp.platform.extension}/ + META-INF/native-image/ META-INF/MANIFEST.MF-* @@ -433,6 +434,8 @@ Import-Package: \ -classpath ${project.build.outputDirectory} -copylibs + -configdir + ${project.build.outputDirectory}/META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/ -properties ${javacpp.platform}${javacpp.platform.suffix} -Dplatform.root=${javacpp.platform.root} @@ -465,6 +468,7 @@ Import-Package: \ ${javacpp.platform.library.path}/ ${javacpp.platform}${javacpp.platform.extension}/ org/bytedeco/javacpp/${javacpp.platform}${javacpp.platform.extension}/ + META-INF/native-image/${javacpp.platform}${javacpp.platform.extension}/ org/bytedeco/javacpp/windows-*/*.exp diff --git a/src/main/java/org/bytedeco/javacpp/tools/BuildMojo.java b/src/main/java/org/bytedeco/javacpp/tools/BuildMojo.java index f77a11b43..a1b19cc27 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/BuildMojo.java +++ b/src/main/java/org/bytedeco/javacpp/tools/BuildMojo.java @@ -176,6 +176,10 @@ public class BuildMojo extends AbstractMojo { @Parameter(property = "javacpp.copyResources", defaultValue = "false") boolean copyResources = false; + /** Also create config files for GraalVM native-image in directory. */ + @Parameter(property = "javacpp.configDirectory") + String configDirectory = null; + /** Also create a JAR file named {@code -.jar}. */ @Parameter(property = "javacpp.jarPrefix") String jarPrefix = null; @@ -280,6 +284,7 @@ String[] merge(String[] ss, String s) { log.debug("header: " + header); log.debug("copyLibs: " + copyLibs); log.debug("copyResources: " + copyResources); + log.debug("configDirectory: " + configDirectory); log.debug("jarPrefix: " + jarPrefix); log.debug("properties: " + properties); log.debug("propertyFile: " + propertyFile); @@ -328,6 +333,7 @@ String[] merge(String[] ss, String s) { .header(header) .copyLibs(copyLibs) .copyResources(copyResources) + .configDirectory(configDirectory) .jarPrefix(jarPrefix) .properties(properties) .propertyFile(propertyFile) diff --git a/src/main/java/org/bytedeco/javacpp/tools/Builder.java b/src/main/java/org/bytedeco/javacpp/tools/Builder.java index 3fcfab17c..e7e279313 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/Builder.java +++ b/src/main/java/org/bytedeco/javacpp/tools/Builder.java @@ -28,6 +28,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStreamReader; +import java.io.PrintWriter; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -547,6 +548,8 @@ File[] generateAndCompile(Class[] classes, String outputName, boolean first, boo Generator generator = new Generator(logger, properties, encoding); String[] sourceFilenames = {sourcePrefixes[0] + "jnijavacpp" + sourceSuffix, sourcePrefixes[1] + outputName + sourceSuffix}; + String[] configDirectories = {configDirectory != null ? new File(configDirectory, "jnijavacpp").getPath() : null, + configDirectory != null ? new File(configDirectory, outputName).getPath() : null}; String[] headerFilenames = {null, header ? sourcePrefixes[1] + outputName + ".h" : null}; String[] loadSuffixes = {"_jnijavacpp", null}; String[] baseLoadSuffixes = {null, "_jnijavacpp"}; @@ -563,21 +566,26 @@ File[] generateAndCompile(Class[] classes, String outputName, boolean first, boo if (outputName.equals("jnijavacpp")) { // generate a single file if the user only wants "jnijavacpp" sourceFilenames = new String[] {sourcePrefixes[0] + outputName + sourceSuffix}; + configDirectories = new String[] {configDirectory != null ? new File(configDirectory, outputName).getPath() : null}; headerFilenames = new String[] {header ? sourcePrefixes[0] + outputName + ".h" : null}; loadSuffixes = new String[] {null}; baseLoadSuffixes = new String[] {null}; - classPaths = new String[] {null}; - classesArray = new Class[][] {null}; + classPaths = new String[] {classPath}; + classesArray = new Class[][] {classes}; libraryNames = new String[] {libraryPrefix + outputName + librarySuffix}; } boolean generated = true; + String[] jniConfigFilenames = new String[sourceFilenames.length]; + String[] reflectConfigFilenames = new String[sourceFilenames.length]; for (int i = 0; i < sourceFilenames.length; i++) { if (i == 0 && !first) { continue; } logger.info("Generating " + sourceFilenames[i]); - if (!generator.generate(sourceFilenames[i], headerFilenames[i], + jniConfigFilenames[i] = configDirectories[i] != null ? configDirectories[i] + File.separator + "jni-config.json" : null; + reflectConfigFilenames[i] = configDirectories[i] != null ? configDirectories[i] + File.separator + "reflect-config.json" : null; + if (!generator.generate(sourceFilenames[i], jniConfigFilenames[i], reflectConfigFilenames[i], headerFilenames[i], loadSuffixes[i], baseLoadSuffixes[i], classPaths[i], classesArray[i])) { logger.info("Nothing generated for " + sourceFilenames[i]); generated = false; @@ -625,6 +633,30 @@ File[] generateAndCompile(Class[] classes, String outputName, boolean first, boo outputFiles[i] = new File(sourceFilenames[i]); } } + + if (header) { + for (String headerFilename : headerFilenames) { + if (headerFilename != null) { + outputFiles = Arrays.copyOf(outputFiles, outputFiles.length + 1); + outputFiles[outputFiles.length - 1] = new File(headerFilename); + } + } + } + + if (configDirectory != null) { + for (String jniConfigFilename : jniConfigFilenames) { + if (jniConfigFilename != null) { + outputFiles = Arrays.copyOf(outputFiles, outputFiles.length + 1); + outputFiles[outputFiles.length - 1] = new File(jniConfigFilename); + } + } + for (String reflectConfigFilename : reflectConfigFilenames) { + if (reflectConfigFilename != null) { + outputFiles = Arrays.copyOf(outputFiles, outputFiles.length + 1); + outputFiles[outputFiles.length - 1] = new File(reflectConfigFilename); + } + } + } } return outputFiles; } @@ -706,6 +738,8 @@ public Builder(Logger logger) { /** The name of the output generated source file or shared library. This enables single- * file output mode. By default, the top-level enclosing classes get one file each. */ String outputName = null; + /** The name of the directory where to output config files for GraalVM native-image, if not {@code null}. */ + File configDirectory = null; /** The name of the JAR file to create, if not {@code null}. */ String jarPrefix = null; /** If true, deletes all files from {@link #outputDirectory} before writing anything in it. */ @@ -802,6 +836,16 @@ public Builder outputName(String outputName) { this.outputName = outputName; return this; } + /** Sets the {@link #configDirectory} field to the argument. */ + public Builder configDirectory(String configDirectory) { + configDirectory(configDirectory == null ? null : new File(configDirectory)); + return this; + } + /** Sets the {@link #configDirectory} field to the argument. */ + public Builder configDirectory(File configDirectory) { + this.configDirectory = configDirectory; + return this; + } /** Sets the {@link #jarPrefix} field to the argument. */ public Builder jarPrefix(String jarPrefix) { this.jarPrefix = jarPrefix; @@ -1128,7 +1172,7 @@ public File[] build() throws IOException, InterruptedException, ParserException if (files != null && files.length > 0) { // files[0] might be null if "jnijavacpp" was not generated and compiled - File directory = files[files.length - 1].getParentFile(); + File directory = files[0] != null ? files[0].getParentFile() : files[files.length - 1].getParentFile(); outputFiles.addAll(Arrays.asList(files)); if (copyLibs) { // Do not copy library files from inherit properties ... @@ -1163,6 +1207,8 @@ public File[] build() throws IOException, InterruptedException, ParserException logger.info("Copying " + fi); Files.copy(fi.toPath(), fo.toPath(), StandardCopyOption.REPLACE_EXISTING); outputFiles.add(fo); + files = Arrays.copyOf(files, files.length + 1); + files[files.length - 1] = fo; } } } @@ -1206,6 +1252,29 @@ public File[] build() throws IOException, InterruptedException, ParserException } } } + if (configDirectory != null) { + File file = new File(configDirectory, name + "/resource-config.json"); + File dir = file.getParentFile(); + if (dir != null) { + dir.mkdirs(); + } + logger.info("Generating " + file); + try (PrintWriter out = encoding != null ? new PrintWriter(file, encoding) : new PrintWriter(file)) { + out.println("{"); + out.println(" \"resources\": ["); + out.print(" {\"pattern\": \"META-INF/.*\"}"); + String separator = "," + System.lineSeparator(); + for (File f : files != null ? files : new File[0]) { + if (!f.toPath().startsWith(configDirectory.toPath())) { + out.print(separator + " {\"pattern\": \".*/" + f.getName() + "\"}"); + } + } + out.println(); + out.println(" ]"); + out.println("}"); + } + outputFiles.add(file); + } } File[] files = outputFiles.toArray(new File[outputFiles.size()]); @@ -1251,6 +1320,7 @@ public static void printHelp() { System.out.println(" -header Generate header file with declarations of callbacks functions"); System.out.println(" -copylibs Copy to output directory dependent libraries (link and preload)"); System.out.println(" -copyresources Copy to output directory resources listed in properties"); + System.out.println(" -configdir Also create config files for GraalVM native-image in directory"); System.out.println(" -jarprefix Also create a JAR file named \"-.jar\""); System.out.println(" -properties Load all platform properties from resource"); System.out.println(" -propertyfile Load all platform properties from file"); @@ -1304,6 +1374,8 @@ public static void main(String[] args) throws Exception { builder.copyLibs(true); } else if ("-copyresources".equals(args[i])) { builder.copyResources(true); + } else if ("-configdir".equals(args[i])) { + builder.configDirectory(args[++i]); } else if ("-jarprefix".equals(args[i])) { builder.jarPrefix(args[++i]); } else if ("-properties".equals(args[i])) { diff --git a/src/main/java/org/bytedeco/javacpp/tools/Generator.java b/src/main/java/org/bytedeco/javacpp/tools/Generator.java index a4a890d83..8ff1513b5 100644 --- a/src/main/java/org/bytedeco/javacpp/tools/Generator.java +++ b/src/main/java/org/bytedeco/javacpp/tools/Generator.java @@ -170,15 +170,15 @@ static enum LongEnum { LONG; long value; } final Logger logger; final Properties properties; final String encoding; - PrintWriter out, out2; + PrintWriter out, out2, jniConfigOut, reflectConfigOut; Map callbacks; IndexedSet functions, deallocators, arrayDeallocators, jclasses; Map> members, virtualFunctions, virtualMembers; Map annotationCache; boolean mayThrowExceptions, usesAdapters, passesStrings, accessesEnums; - public boolean generate(String sourceFilename, String headerFilename, String loadSuffix, - String baseLoadSuffix, String classPath, Class ... classes) throws IOException { + public boolean generate(String sourceFilename, String jniConfigFilename, String reflectConfigFilename, String headerFilename, + String loadSuffix, String baseLoadSuffix, String classPath, Class ... classes) throws IOException { try { // first pass using a null writer to fill up the IndexedSet objects out = new PrintWriter(new Writer() { @@ -186,7 +186,7 @@ public boolean generate(String sourceFilename, String headerFilename, String loa @Override public void flush() { } @Override public void close() { } }); - out2 = null; + out2 = jniConfigOut = reflectConfigOut= null; callbacks = new LinkedHashMap(); functions = new IndexedSet(); deallocators = new IndexedSet(); @@ -221,6 +221,24 @@ public boolean generate(String sourceFilename, String headerFilename, String loa } out2 = encoding != null ? new PrintWriter(headerFile, encoding) : new PrintWriter(headerFile); } + if (jniConfigFilename != null) { + logger.info("Generating " + jniConfigFilename); + File jniConfigFile = new File(jniConfigFilename); + File jniConfigDir = jniConfigFile.getParentFile(); + if (jniConfigDir != null) { + jniConfigDir.mkdirs(); + } + jniConfigOut = encoding != null ? new PrintWriter(jniConfigFile, encoding) : new PrintWriter(jniConfigFile); + } + if (reflectConfigFilename != null) { + logger.info("Generating " + reflectConfigFilename); + File reflectConfigFile = new File(reflectConfigFilename); + File reflectConfigDir = reflectConfigFile.getParentFile(); + if (reflectConfigDir != null) { + reflectConfigDir.mkdirs(); + } + reflectConfigOut = encoding != null ? new PrintWriter(reflectConfigFile, encoding) : new PrintWriter(reflectConfigFile); + } return classes(mayThrowExceptions, usesAdapters, passesStrings, accessesEnums, loadSuffix, baseLoadSuffix, classPath, classes); } else { return false; @@ -232,6 +250,12 @@ public boolean generate(String sourceFilename, String headerFilename, String loa if (out2 != null) { out2.close(); } + if (jniConfigOut != null) { + jniConfigOut.close(); + } + if (reflectConfigOut != null) { + reflectConfigOut.close(); + } } } @@ -1709,6 +1733,44 @@ boolean classes(boolean handleExceptions, boolean defineAdapters, boolean conver out2.println("#endif"); } + for (PrintWriter o : new PrintWriter[] {jniConfigOut, reflectConfigOut}) { + allClasses.addAll(jclasses.keySet()); + + LinkedHashSet reflectClasses = new LinkedHashSet(); + reflectClasses.addAll(baseClasses); + reflectClasses.add(Object.class); + reflectClasses.add(Buffer.class); + reflectClasses.add(String.class); + + if (o != null) { + o.println("["); + String separator = ""; + for (Class cls : allClasses) { + do { + o.println(separator + " {"); + o.print(" \"name\" : \"" + cls.getName() + "\""); + if (reflectClasses.contains(cls) || reflectClasses.contains(cls.getEnclosingClass())) { + o.println(","); + o.println(" \"allDeclaredConstructors\" : true,"); + o.println(" \"allPublicConstructors\" : true,"); + o.println(" \"allDeclaredMethods\" : true,"); + o.println(" \"allPublicMethods\" : true,"); + o.println(" \"allDeclaredFields\" : true,"); + o.println(" \"allPublicFields\" : true,"); + o.println(" \"allDeclaredClasses\" : true,"); + o.print(" \"allPublicClasses\" : true"); + } + o.println(); + o.print(" }"); + separator = "," + System.lineSeparator(); + cls = cls.getEnclosingClass(); + } while (cls != null); + } + o.println(); + o.println("]"); + } + } + return supportedPlatform; }