diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f844161d0..b886b25a5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -40,6 +40,11 @@ jobs: matrix: java: [21, 23] + env: + CC: gcc10-gcc + CXX: gcc10-g++ + FC: gcc10-gfortran + name: Build and Test k-NN Plugin on Linux runs-on: ubuntu-latest needs: Get-CI-Image-Tag diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 4cfbbf47c..0845c792a 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -39,6 +39,10 @@ jobs: strategy: matrix: java: [21] + env: + CC: gcc10-gcc + CXX: gcc10-g++ + FC: gcc10-gfortran name: Run Integration Tests on Linux runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 827b8f9ea..a290784bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Upgrade min JDK compatibility to JDK 21 [#2422](https://github.com/opensearch-project/k-NN/pull/2422) ### Documentation ### Maintenance +* Update package name to fix compilation issue [#2513](https://github.com/opensearch-project/k-NN/pull/2513) ### Refactoring ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.18...2.x) diff --git a/build.gradle b/build.gradle index c440922af..0a48aa6f3 100644 --- a/build.gradle +++ b/build.gradle @@ -405,22 +405,36 @@ tasks.register('cmakeJniLib', Exec) { standardOutput = outputStream } +task buildNmslib(type:Exec) { + dependsOn cmakeJniLib + def makePath = findExecutable("make") + logger.lifecycle("Using make at: ${makePath}") + // Ensure makePath is treated as a String. If parsing to an int is required, + // handle it explicitly, though a path typically should not be parsed as an int. + if (makePath.isEmpty()) { + throw new GradleException("Make not found in PATH. Please install Make.") + } + def outputStream = new ByteArrayOutputStream() + commandLine makePath, '-Cjni/build', 'opensearchknn_nmslib', '-j', "${nproc_count}" + standardOutput = outputStream +} + task buildJniLib(type:Exec) { dependsOn cmakeJniLib def makePath = findExecutable("make") logger.lifecycle("Using make at: ${makePath}") - println "Make path: ${makePath}" // Ensure makePath is treated as a String. If parsing to an int is required, // handle it explicitly, though a path typically should not be parsed as an int. if (makePath.isEmpty()) { throw new GradleException("Make not found in PATH. Please install Make.") } def outputStream = new ByteArrayOutputStream() - commandLine makePath, '-Cjni/build','opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}" + commandLine makePath, '-Cjni/build', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}" standardOutput = outputStream } test { + dependsOn buildNmslib dependsOn buildJniLib systemProperty 'tests.security.manager', 'false' systemProperty "java.library.path", "$rootDir/jni/release" @@ -435,6 +449,7 @@ test { def _numNodes = findProperty('numNodes') as Integer ?: 1 integTest { if (integTestDependOnJniLib) { + dependsOn buildNmslib dependsOn buildJniLib } systemProperty 'tests.security.manager', 'false' @@ -533,6 +548,7 @@ task integTestRemote(type: RestIntegTestTask) { run { useCluster project.testClusters.integTest + dependsOn buildNmslib dependsOn buildJniLib doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 3dbf151a7..b444f6af7 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -135,6 +135,7 @@ testClusters { // All nodes are upgraded to latest version and run the tests task testRestartUpgrade(type: StandaloneRestIntegTestTask) { dependsOn "testAgainstOldCluster" + dependsOn rootProject.tasks.buildNmslib dependsOn rootProject.tasks.buildJniLib dependsOn rootProject.tasks.assemble useCluster testClusters."${baseName}" diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 9c7940617..13145a5f3 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -52,6 +52,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}" + dependsOn rootProject.tasks.buildNmslib dependsOn rootProject.tasks.buildJniLib dependsOn rootProject.tasks.assemble dependsOn "testAgainstOldCluster" diff --git a/scripts/build.sh b/scripts/build.sh index a31daad94..92be92b4c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -133,9 +133,14 @@ cd $work_dir ./gradlew :buildJniLib -Davx512.enabled=false -Davx512_spr.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1} if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then + echo "Building k-NN library nmslib with gcc 10 on non-windows x64" + rm -rf jni/CMakeCache.txt jni/CMakeFiles + env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false + echo "Building k-NN library after enabling AVX2" # Skip applying patches as patches were applied already from previous :buildJniLib task # If we apply patches again, it fails with conflict + rm -rf jni/CMakeCache.txt jni/CMakeFiles ./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false echo "Building k-NN library after enabling AVX512" @@ -143,6 +148,9 @@ if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library after enabling AVX512_SPR" ./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false + +else + ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false fi ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER @@ -151,7 +159,7 @@ fi # Add lib to zip zipPath=$(find "$(pwd)/build/distributions" -path \*.zip) distributions="$(dirname "${zipPath}")" -mkdir $distributions/lib +mkdir -p $distributions/lib libPrefix="libopensearchknn" if [ "$PLATFORM" = "windows" ]; then libPrefix="opensearchknn"