Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for python env managers #1010

Open
wants to merge 16 commits into
base: master
Choose a base branch
from
Open

Support for python env managers #1010

Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
3a74bcf
Support conda and venv
lfoppiano May 6, 2023
2a8e5a8
cleanup
lfoppiano May 7, 2023
89a91a4
fix librarypath, remove superfluous colon
lfoppiano May 8, 2023
1ac9812
Merge branch 'master' into python-envs
lfoppiano Mar 12, 2024
a7c0eea
Merge remote-tracking branch 'origin/python-envs' into python-envs
lfoppiano Jun 17, 2024
e5a2c59
Merge branch 'refs/heads/release-0.8.1' into python-envs
lfoppiano Sep 8, 2024
5ab4f91
update
lfoppiano Sep 8, 2024
08319b2
fix missing var
lfoppiano Sep 8, 2024
4b7da7f
fix print
lfoppiano Sep 8, 2024
b97e70f
revise the gradle stuff
lfoppiano Sep 8, 2024
ac19f81
cleanup
lfoppiano Sep 8, 2024
a9425d2
make it a string
lfoppiano Sep 8, 2024
fc769b6
transform the java path var as string
lfoppiano Sep 9, 2024
bd426c4
Merge branch 'master' into python-envs
lfoppiano Sep 14, 2024
37cb3d1
Merge branch 'master' into python-envs
lfoppiano Nov 29, 2024
8a16401
cosmetics in case of JVM 8
lfoppiano Nov 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 53 additions & 47 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,40 @@ repositories {
}
}

def getJavaLibraryPath = {
def jepLocalLibraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
if (Os.OS_ARCH.equals("aarch64")) {
jepLocalLibraries = "${file("./grobid-home/lib/mac_arm-64").absolutePath}"
} else {
jepLocalLibraries = "${file("./grobid-home/lib/mac-64").absolutePath}"
}
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
jepLocalLibraries = "${rootProject.rootDir.getAbsolutePath()}/grobid-home/lib/lin-64/jep:" +
"${rootProject.rootDir.getAbsolutePath()}/grobid-home/lib/lin-64"
} else {
throw new RuntimeException("Unsupported platform!")
}

def javaLibraryPath = "${System.getProperty('java.library.path')}:${jepLocalLibraries}"

def envPrefix = ""
if (System.env.CONDA_PREFIX) {
envPrefix = "${System.env.CONDA_PREFIX}"
} else if (System.env.VIRTUAL_ENV) {
envPrefix = "${System.env.VIRTUAL_ENV}"
}

if (envPrefix) {
def envLibs = "${envPrefix}/lib"
def pythonDirectory = file(envLibs).listFiles({ it.toString().contains("/lib/python") } as FileFilter)?.first()
def pythonVersion = (pythonDirectory =~ /python([0-9]\.[0-9]+)/)[0][1]

javaLibraryPath += ":${envLibs}:${envLibs}/python${pythonVersion}/site-packages/jep"
}
return javaLibraryPath
}

apply plugin: 'jacoco'

jacoco {
Expand Down Expand Up @@ -192,27 +226,13 @@ subprojects {
forkEvery = 1
maxHeapSize = "1024m"

def libraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
if (Os.OS_ARCH.equals("aarch64")) {
libraries = "${file("./grobid-home/lib/mac_arm-64").absolutePath}"
} else {
libraries = "${file("./grobid-home/lib/mac-64").absolutePath}"
}
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
def jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64/jep"
libraries = jepDir
jepDir = rootProject.rootDir.getAbsolutePath() + "/grobid-home/lib/lin-64"
libraries += ":"+jepDir
} else {
throw new RuntimeException("Unsupported platform!")
}
def javaLibraryPath = getJavaLibraryPath()

if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs "--add-opens", "java.base/java.util.stream=ALL-UNNAMED",
"--add-opens", "java.base/java.io=ALL-UNNAMED", "--add-opens", "java.xml/jdk.xml.internal=ALL-UNNAMED"
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
systemProperty "java.library.path", "${javaLibraryPath}"
}
}

Expand Down Expand Up @@ -338,36 +358,14 @@ project(":grobid-service") {
mainClassName = 'org.grobid.service.main.GrobidServiceApplication'

tasks.run {
def libraries = ""
if (Os.isFamily(Os.FAMILY_MAC)) {
if (Os.OS_ARCH.equals("aarch64")) {
libraries = "${file("../grobid-home/lib/mac_arm-64").absolutePath}"
} else {
libraries = "${file("../grobid-home/lib/mac-64").absolutePath}"
}
} else if (Os.isFamily(Os.FAMILY_UNIX)) {
libraries = "${file("../grobid-home/lib/lin-64/jep").absolutePath}:" +
"${file("../grobid-home/lib/lin-64").absolutePath}:"
} else {
throw new RuntimeException("Unsupported platform!")
}
def javaLibraryPath = getJavaLibraryPath()

if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs "--add-opens", "java.base/java.lang=ALL-UNNAMED"
}
workingDir = rootProject.rootDir
def javaLibraryPath = "${System.getProperty('java.library.path')}:" + libraries
// if (System.env.CONDA_PREFIX) {
// def condaEnv = "${System.env.CONDA_PREFIX}/lib"
// def pythonDirectory = file(condaEnv).listFiles({ it.toString().contains("/lib/python") } as FileFilter)?.first()
// def pythonVersion = (pythonDirectory =~ /python([0-9]\.[0-9]+)/)[0][1]
//
// javaLibraryPath = "${System.getProperty('java.library.path')}:" +
// libraries + ":" +
// "${System.env.CONDA_PREFIX}/lib:" +
// "${System.env.CONDA_PREFIX}/lib/python${pythonVersion}/site-packages/jep"
// }
systemProperty "java.library.path", javaLibraryPath

systemProperty "java.library.path", "${javaLibraryPath}"
}

tasks.distZip.enabled = true
Expand Down Expand Up @@ -535,9 +533,13 @@ project(":grobid-trainer") {
tasks.create(name: taskName, type: JavaExec, group: 'modeltraining') {
main = mainClassName
classpath = sourceSets.main.runtimeClasspath
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0)
def javaLibraryPath = getJavaLibraryPath()
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path", "${javaLibraryPath}"
}
}

Expand All @@ -556,48 +558,52 @@ project(":grobid-trainer") {
main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
classpath = sourceSets.main.runtimeClasspath
args 'nlm', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
def javaLibraryPath = getJavaLibraryPath()
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
systemProperty "java.library.path", "${javaLibraryPath}"
}

task(teiEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
classpath = sourceSets.main.runtimeClasspath
args 'tei', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
def javaLibraryPath = getJavaLibraryPath()
if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
systemProperty "java.library.path", "${javaLibraryPath}"
}

task(PrepareDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
classpath = sourceSets.main.runtimeClasspath
args 'data', getArg('p2t', '.')
def javaLibraryPath = getJavaLibraryPath()
if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
systemProperty "java.library.path", "${javaLibraryPath}"
}

task(EvaluateDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
classpath = sourceSets.main.runtimeClasspath
args 'eval', getArg('p2t', '.')
def javaLibraryPath = getJavaLibraryPath()
if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
} else {
jvmArgs '-Xmx3072m'
}
systemProperty "java.library.path","${System.getProperty('java.library.path')}:" + libraries
systemProperty "java.library.path", "${javaLibraryPath}"
}
}

Expand Down