Update build.gradle and build.sh to separate x64 linux nmslib build w…

…ith different gcc versions (#2506) * Update build gradle to separate nmslib / faiss generation Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Update scripts/build.sh to separate enable gcc switch Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Remove test comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Remove test comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Remove test comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Updating restart and rolling upgrade bwc test bundle.gradle Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Enforce gcc10 for nmslib to compile and avx512_spr have no-op Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> --------- Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · Feb 10, 2025 · a4bea6f · a4bea6f
1 parent 905c9cb
commit a4bea6f
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 2 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -40,6 +40,11 @@ jobs:
       matrix:
         java: [21, 23]
 
+    env:
+       CC: gcc10-gcc
+       CXX: gcc10-g++
+       FC: gcc10-gfortran
+
     name: Build and Test k-NN Plugin on Linux
     runs-on: ubuntu-latest
     needs: Get-CI-Image-Tag

diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -39,6 +39,10 @@ jobs:
     strategy:
       matrix:
         java: [21]
+    env:
+       CC: gcc10-gcc
+       CXX: gcc10-g++
+       FC: gcc10-gfortran
 
     name: Run Integration Tests on Linux
     runs-on: ubuntu-latest

diff --git a/build.gradle b/build.gradle
@@ -356,13 +356,20 @@ task cmakeJniLib(type:Exec) {
     commandLine args
 }
 
+task buildNmslib(type:Exec) {
+    dependsOn cmakeJniLib
+    workingDir 'jni'
+    commandLine 'make', 'opensearchknn_nmslib', '-j', "${nproc_count}"
+}
+
 task buildJniLib(type:Exec) {
     dependsOn cmakeJniLib
     workingDir 'jni'
-    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
+    commandLine 'make', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
 }
 
 test {
+    dependsOn buildNmslib
     dependsOn buildJniLib
     systemProperty 'tests.security.manager', 'false'
     systemProperty "java.library.path", "$rootDir/jni/release"
@@ -377,6 +384,7 @@ test {
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 integTest {
     if (integTestDependOnJniLib) {
+        dependsOn buildNmslib
         dependsOn buildJniLib
     }
     systemProperty 'tests.security.manager', 'false'
@@ -475,6 +483,7 @@ task integTestRemote(type: RestIntegTestTask) {
 
 run {
     useCluster project.testClusters.integTest
+    dependsOn buildNmslib
     dependsOn buildJniLib
     doFirst {
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts

diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle
@@ -135,6 +135,7 @@ testClusters {
 // All nodes are upgraded to latest version and run the tests
     task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
         dependsOn "testAgainstOldCluster"
+        dependsOn rootProject.tasks.buildNmslib
         dependsOn rootProject.tasks.buildJniLib
         dependsOn rootProject.tasks.assemble
         useCluster testClusters."${baseName}"

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
@@ -52,6 +52,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}"
+    dependsOn rootProject.tasks.buildNmslib
     dependsOn rootProject.tasks.buildJniLib
     dependsOn rootProject.tasks.assemble
     dependsOn "testAgainstOldCluster"

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -133,16 +133,24 @@ cd $work_dir
 ./gradlew :buildJniLib -Davx512.enabled=false -Davx512_spr.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1}
 
 if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then
+  echo "Building k-NN library nmslib with gcc 10 on non-windows x64"
+  rm -rf jni/CMakeCache.txt jni/CMakeFiles
+  env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
+
   echo "Building k-NN library after enabling AVX2"
   # Skip applying patches as patches were applied already from previous :buildJniLib task
   # If we apply patches again, it fails with conflict
+  rm -rf jni/CMakeCache.txt jni/CMakeFiles
   ./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 
   echo "Building k-NN library after enabling AVX512"
   ./gradlew :buildJniLib -Davx512.enabled=true -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 
   echo "Building k-NN library after enabling AVX512_SPR"
   ./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
+
+else
+  ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 fi
 
 ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
@@ -151,7 +159,7 @@ fi
 # Add lib to zip
 zipPath=$(find "$(pwd)/build/distributions" -path \*.zip)
 distributions="$(dirname "${zipPath}")"
-mkdir $distributions/lib
+mkdir -p $distributions/lib
 libPrefix="libopensearchknn"
 if [ "$PLATFORM" = "windows" ]; then
     libPrefix="opensearchknn"