Added support to query local arrays

.github/workflows/everything.yml

@@ -448,24 +448,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        code: [examples, lammps, tau]
+        code: [lammps, tau]
         include:
-          - code: examples
-            repo: ornladios/ADIOS2-Examples
-            ref: master
           - code: lammps
             repo: pnorbert/lammps
             ref: fix-deprecated-adios-init
           - code: tau
-            repo: ornladios/ADIOS2-Examples
-            ref: master
-
     defaults:
       run:
         shell: bash -c "docker exec adios2-ci bash --login -e $(echo {0} | sed 's|/home/runner/work|/__w|g')"
 
     steps:
       - uses: actions/checkout@v4
+        if: ${{ matrix.repo != '' }}
         with:
           repository: ${{ matrix.repo }}
           ref: ${{ matrix.ref }}

.gitlab/config/SpackCIBridge.py

@@ -169,7 +169,7 @@ def list_github_prs(self):
                         # Check if we should defer pushing/testing this PR because it is based on "too new" of a commit
                         # of the main branch.
                         tmp_pr_branch = f"temporary_{pr_string}"
-                        subprocess.run(["git", "fetch", "--unshallow", "github",
+                        subprocess.run(["git", "fetch", "github",
                                        f"refs/pull/{pull.number}/head:{tmp_pr_branch}"], check=True)
                         # Get the merge base between this PR and the main branch.
                         try:
@@ -226,7 +226,7 @@ def list_github_prs(self):
                         # then we will push the merge commit that was automatically created by GitHub to GitLab
                         # where it will kick off a CI pipeline.
                         try:
-                            subprocess.run(["git", "fetch", "--unshallow", "github",
+                            subprocess.run(["git", "fetch", "github",
                                            f"{pull.merge_commit_sha}:{pr_string}"], check=True)
                         except subprocess.CalledProcessError:
                             print("Failed to locally checkout PR {0} ({1}). Skipping"
@@ -306,7 +306,7 @@ def setup_git_repo(self):
         self.gitlab_shallow_fetch()
 
         if self.main_branch:
-            subprocess.run(["git", "fetch", "--unshallow", "github", self.main_branch], check=True)
+            subprocess.run(["git", "fetch", "github", self.main_branch], check=True)
 
     def get_gitlab_pr_branches(self):
         """Query GitLab for branches that have already been copied over from GitHub PRs.
@@ -350,7 +350,7 @@ def update_refspecs_for_tags(self, tags, open_refspecs, fetch_refspecs):
     def fetch_github_branches(self, fetch_refspecs):
         """Perform `git fetch` for a given list of refspecs."""
         print("Fetching GitHub refs for open PRs")
-        fetch_args = ["git", "fetch", "-q", "--unshallow", "github"] + fetch_refspecs
+        fetch_args = ["git", "fetch", "-q", "github"] + fetch_refspecs
         subprocess.run(fetch_args, check=True)
 
     def build_local_branches(self, protected_branches):

CMakeLists.txt

@@ -137,7 +137,11 @@ if((NOT BUILD_SHARED_LIBS) AND (NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE))
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 endif()
 
+# Ctest creates BUILD_TESTING option and sets it to true by default
+# Here we disable BUILD_TESTING option by default
+option(BUILD_TESTING "Build testing" OFF)
 include(CTest)
+mark_as_advanced(CLEAR BUILD_TESTING)
 
 adios_option(Blosc2     "Enable support for c-blosc-2 transforms" AUTO)
 adios_option(BZip2      "Enable support for BZip2 transforms" AUTO)
@@ -176,7 +180,7 @@ if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA)
         if(DEFINED Kokkos_CUDA_ARCHITECTURES)
             set(CMAKE_CUDA_ARCHITECTURES ${Kokkos_CUDA_ARCHITECTURES})
         else()
-            # Mininum common non-deprecated architecture
+            # Minimum common non-deprecated architecture
             set(CMAKE_CUDA_ARCHITECTURES 52)
         endif()
     endif()
@@ -296,8 +300,6 @@ endif()
 #------------------------------------------------------------------------------#
 # Third party libraries
 #------------------------------------------------------------------------------#
-include(CTest)
-mark_as_advanced(BUILD_TESTING)
 add_subdirectory(thirdparty)
 
 #------------------------------------------------------------------------------#
@@ -318,7 +320,7 @@ add_subdirectory(plugins)
 #------------------------------------------------------------------------------#
 # Examples
 #------------------------------------------------------------------------------#
-option(ADIOS2_BUILD_EXAMPLES "Build examples" ON)
+option(ADIOS2_BUILD_EXAMPLES "Build examples" OFF)
 if(ADIOS2_BUILD_EXAMPLES)
   add_subdirectory(examples)
 endif()

ReadMe.md

@@ -78,7 +78,6 @@ Once ADIOS2 is installed refer to:
 
 ADIOS2 is an open source project: Questions, discussion, and contributions are welcome. Join us at:
 
-- Slack workspace: [![Slack](https://img.shields.io/badge/slack-ADIOS2-pink.svg)](https://adios2.spack.io) 
 - Mailing list: adios-ecp@kitware.com 
 - Github Discussions: https://github.com/ornladios/ADIOS2/discussions
 

bindings/CXX11/adios2/cxx11/Query.cpp

@@ -14,6 +14,11 @@ QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader)
     delete m;
 }
 
+void QueryWorker::GetResultCoverage(std::vector<size_t> &touched_blockIDs)
+{
+    m_Worker->GetResultCoverage(touched_blockIDs);
+}
+
 void QueryWorker::GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks)
 {
     adios2::Box<adios2::Dims> empty;
@@ -26,4 +31,5 @@ void QueryWorker::GetResultCoverage(const adios2::Box<adios2::Dims> &outputSelec
     if (m_Worker)
         return m_Worker->GetResultCoverage(outputSelection, touched_blocks);
 }
-}
+
+} // namespace

bindings/CXX11/adios2/cxx11/Query.h

@@ -29,6 +29,7 @@ class QueryWorker
     // configFile has query, can be either xml or json
     QueryWorker(const std::string &configFile, adios2::Engine &engine);
 
+    void GetResultCoverage(std::vector<size_t> &touched_block_ids);
     // touched_blocks is a list of regions specified by (start, count),
     // that contains data that satisfies the query file
     void GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks);

bindings/Matlab/README.txt

@@ -43,5 +43,5 @@ $ cd <your adios source>
 $ git pull
 $ mkdir build.lean
 $ cd build.lean
-$ cmake -DCMAKE_INSTALL_PREFIX=/opt/adios/lean -DADIOS2_USE_MPI=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo -DADIOS2_ENABLE_PIC=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_TESTING=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_BZip2=OFF -DADIOS2_USE_DataMan=OFF  ..
+$ cmake -DCMAKE_INSTALL_PREFIX=/opt/adios/lean -DADIOS2_USE_MPI=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo -DADIOS2_ENABLE_PIC=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DADIOS2_USE_SST=OFF -DADIOS2_USE_BZip2=OFF -DADIOS2_USE_DataMan=OFF  ..
 

bindings/Python/py11Query.cpp

@@ -36,5 +36,12 @@ std::vector<Box<Dims>> Query::GetResult()
     return touched_blocks;
 }
 
+std::vector<size_t> Query::GetBlockIDs()
+{
+    std::vector<size_t> touched_block_ids;
+    m_QueryWorker->GetResultCoverage(touched_block_ids);
+    return touched_block_ids;
+}
+
 } // py11
 } // adios2

bindings/Python/py11Query.h

@@ -34,6 +34,7 @@ class Query
     explicit operator bool() const noexcept;
 
     std::vector<Box<Dims>> GetResult();
+    std::vector<size_t> GetBlockIDs();
 
 private:
     Query(adios2::query::Worker *qw);

bindings/Python/py11glue.cpp

@@ -357,7 +357,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
              "adios2 query construction, a xml query File and a read engine",
              pybind11::arg("queryFile"), pybind11::arg("reader") = true)
 
-        .def("GetResult", &adios2::py11::Query::GetResult);
+        .def("GetResult", &adios2::py11::Query::GetResult)
+        .def("GetBlockIDs", &adios2::py11::Query::GetBlockIDs);
 
     pybind11::class_<adios2::py11::Variable>(m, "Variable")
         // Python 2

cmake/DetectOptions.cmake

@@ -76,7 +76,7 @@ if(ADIOS2_USE_Blosc2 STREQUAL AUTO)
   endif()
 elseif(ADIOS2_USE_Blosc2)
   # Prefect CONFIG mode
-  find_package(Blosc2 2.4 CONFIG REQUIRED)
+  find_package(Blosc2 2.4 CONFIG)
   if(NOT Blosc2_FOUND)
     find_package(Blosc2 2.4 MODULE REQUIRED)
   endif()
@@ -216,7 +216,7 @@ if(ADIOS2_USE_Kokkos)
         enable_language(HIP)
       endif()
       if(Kokkos_ENABLE_SYCL)
-	    set(ADIOS2_HAVE_Kokkos_SYCL TRUE)
+        set(ADIOS2_HAVE_Kokkos_SYCL TRUE)
       endif()
       set(ADIOS2_HAVE_GPU_Support TRUE)
     endif()
@@ -445,9 +445,11 @@ if(ADIOS2_USE_SST AND NOT WIN32)
     endif()
   endif()
   if(ADIOS2_HAVE_MPI)
-    set(CMAKE_REQUIRED_LIBRARIES MPI::MPI_C)
-    include(CheckCSourceRuns)
-    check_c_source_runs([=[
+    set(CMAKE_REQUIRED_LIBRARIES "MPI::MPI_C;Threads::Threads")
+    include(CheckCXXSourceRuns)
+    check_cxx_source_runs([=[
+        #include <chrono>
+        #include <future>
         #include <mpi.h>
         #include <stdlib.h>
 
@@ -457,9 +459,18 @@ if(ADIOS2_USE_SST AND NOT WIN32)
 
         int main()
         {
+          // Timeout after 5 second
+          auto task = std::async(std::launch::async, []() {
+                                 std::this_thread::sleep_for(std::chrono::seconds(5));
+                                 exit(EXIT_FAILURE);
+          });
+
+          char* port_name = new char[MPI_MAX_PORT_NAME];
           MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, NULL);
-          MPI_Open_port(MPI_INFO_NULL, malloc(sizeof(char) * MPI_MAX_PORT_NAME));
+          MPI_Open_port(MPI_INFO_NULL, port_name);
+          MPI_Close_port(port_name);
           MPI_Finalize();
+          exit(EXIT_SUCCESS);
         }]=]
     ADIOS2_HAVE_MPI_CLIENT_SERVER)
     unset(CMAKE_REQUIRED_LIBRARIES)

cmake/adios2-config-install.cmake.in

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.12)
 
 set(_ADIOS2_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH})
 list(INSERT CMAKE_MODULE_PATH 0 "${CMAKE_CURRENT_LIST_DIR}")

cmake/install/post/adios2-config-dummy/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.12)
 
 project(adios2-config-dummy C CXX)
 

cmake/install/post/generate-adios2-config.sh.in

@@ -64,6 +64,12 @@ if [ "${1:0:1}" != "/" ]
 then
   # Convert relative paths to absolute based on DESTDIR
   PREFIX="${DESTDIR:-$(pwd)}/$1"
+else
+  # check if the PREFIX directory exists and prepend DESTDIR if not
+  if [ ! -d "${PREFIX}" ]
+  then
+    PREFIX="${DESTDIR}$1"
+  fi
 fi
 
 shift

docs/user_guide/source/setting_up/source/cmake.rst

@@ -103,8 +103,8 @@ In addition to the ``ADIOS2_USE_Feature`` options, the following options are als
  CMake VAR Options                       Values                                       Description                                                                          |
 ==================================== =============================================== ===============================
 ``BUILD_SHARED_LIBS``                  **ON**/OFF                                     Build shared libraries.
-``ADIOS2_BUILD_EXAMPLES``              **ON**/OFF                                     Build examples.
-``BUILD_TESTING``                      **ON**/OFF                                     Build test code.
+``ADIOS2_BUILD_EXAMPLES``              ON/**OFF**                                     Build examples.
+``BUILD_TESTING``                      ON/**OFF**                                     Build test code.
 ``CMAKE_INSTALL_PREFIX``               /path/to/install (``/usr/local``)              Installation location.
 ``CMAKE_BUILD_TYPE``                   Debug/**Release**/RelWithDebInfo/MinSizeRel    Compiler optimization levels.
 ==================================== =============================================== ===============================

examples/CMakeLists.txt

@@ -5,24 +5,6 @@
 
 add_subdirectory(basics)
 add_subdirectory(hello)
-add_subdirectory(query)
-add_subdirectory(useCases)
-add_subdirectory(inlineMWE)
 add_subdirectory(plugins)
-add_subdirectory(fides)
-
-if(ADIOS2_HAVE_MPI)
-  add_subdirectory(heatTransfer)
-endif()
-
-
-if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA)
-  add_subdirectory(cuda)
-endif()
-
-
-if(ADIOS2_HAVE_MPI AND ADIOS2_HAVE_HDF5)
-  if(HDF5_VERSION VERSION_GREATER_EQUAL 1.14)
-    add_subdirectory(h5subfile)
-  endif()
-endif()
+add_subdirectory(simulations)
+add_subdirectory(useCases)

examples/ReadMe.md

@@ -0,0 +1,16 @@
+# ADIOS2 Examples
+
+This directory contains examples of how to use ADIOS2 in different scenarios.
+The examples are written mostly in C++, but also C, Fortran, Python, and Julia.
+
+They can be found in the following subdirectories, and they should be explored in the order that they are listed:
+
+1. [hello](hello): The _hello_ examples are meant to introduce you to ADIOS2's IO capabilities and engines.
+2. [basics](basics): The _basics_ examples are meant to introduce you to basic concepts of ADIOS2, such as
+   global/joined/local arrays, values, and variables shapes.
+3. [useCases](useCases): The _useCases_ examples are meant to demonstrate how to use ADIOS2 in different scenarios,
+   such as in situ visualization, and fides schema.
+4. [plugins](plugins): The _plugins_ examples are meant to introduce you to the plugin capabilities of ADIOS2, such as
+   how to develop your own engine or operators.
+5. [simulations](simulations): The _simulations_ examples are meant to demonstrate how to integrate ADIOS2 within your
+   simulation code to read, write and/or stream your simulation data.

examples/basics/CMakeLists.txt

@@ -3,7 +3,12 @@
 # accompanying file Copyright.txt for details.
 #------------------------------------------------------------------------------#
 
-add_subdirectory(globalArray)
+if(ADIOS2_HAVE_MPI)
+  add_subdirectory(globalArray1D)
+endif()
+add_subdirectory(globalArrayND)
 add_subdirectory(joinedArray)
 add_subdirectory(localArray)
+add_subdirectory(queryWorker)
 add_subdirectory(values)
+add_subdirectory(variablesShapes)

examples/basics/ReadMe.md

@@ -0,0 +1,29 @@
+## ADIOS2 basics examples
+
+The _basics_ examples are meant to introduce you to basic concepts of ADIOS2, such as
+global/joined/local arrays, values, and variables shapes.
+
+They can be found in the following subdirectories, and they should be explored in the order that they are listed:
+
+1. [globalArray1D](globalArray1D): The _globalArray1D_ example demonstrates how to read and write an
+   1-D global array with constant dimensions over time from multiple processors using ADIOS2's BP engine.
+   * Languages: C, Fortran
+2. [globalArrayND](globalArrayND): The _globalArrayND_ example demonstrates how to write an N-D global array with
+   constant dimensions over time from multiple processors using ADIOS2's BP engine.
+   * Languages: C++
+3. [localArray](localArray): The _localArray_ example demonstrates how to write and read a local array per processor
+   with the same name from multiple processors using ADIOS2's BP engine.
+   * Languages: C++
+4. [joinedArray](joinedArray): The _joinedArray_ example demonstrates how to write local array that is different only in
+   one dimension so that it can be joined into a global array with the same name from multiple processors at read time
+   using ADIOS2's ADIOS2's BP engine.
+   * Languages: C++
+5. [values](values): The _values_ example demonstrates how to write and read a multiple types of variables with a single
+   value, such as global constant, global value, local constant, and local value using ADIOS2's BP engine.
+   * Languages: C++, Fortran
+6. [variablesShapes](variablesShapes): The _variablesShapes_ example demonstrates how to write supported variables
+   shapes using stepping and ADIOS2's BP engine.
+   * Languages: C++, C++ using high-level API
+7. [queryWorker](queryWorker): The _queryWorker_ example demonstrates how to read variables using ADIOS2's BP engine
+   and perform queries on the read data and streams the results.
+   * Languages: C++