From f905496cf4a7e0e20a5ee18a726d92300a061daf Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Tue, 1 Jun 2021 23:11:05 -0700 Subject: [PATCH 1/7] Testing commit --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index d5b2943dc..8c4b8db3d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,6 +2,7 @@ sphinx-quickstart on Thu Apr 15 14:28:56 2021. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. + Test - Kenneth Welcome to PDC's documentation! =============================== From 21b7060722b6f2379140dba23251434a9c48626a Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Wed, 2 Jun 2021 09:47:08 -0700 Subject: [PATCH 2/7] Changed theme to sphinx_rtd_theme --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index caaa860ac..646f3e565 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -44,7 +44,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From a76a4d4496ed947f165d9c7af2ae325c49f5f63b Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Wed, 2 Jun 2021 12:51:56 -0700 Subject: [PATCH 3/7] Added css to _static --- docs/source/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 646f3e565..84c224eca 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -49,4 +49,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ['/Users/kenneth/Documents/Berkeley Lab/pdc/docs/_static'] + +def setup(app): + app.add_css_file('theme_overrides.css') From 1c5a88f807f56e40a3e651151729e52d64d0fde2 Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Wed, 2 Jun 2021 12:53:12 -0700 Subject: [PATCH 4/7] Added css to change theme --- docs/_static/theme_overrides.css | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/_static/theme_overrides.css diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css new file mode 100644 index 000000000..40acce4e4 --- /dev/null +++ b/docs/_static/theme_overrides.css @@ -0,0 +1,47 @@ +/* override table width restrictions */ + .wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal !important; + } + .wy-table-responsive { + margin-bottom: 24px; + max-width: 100%; + overflow: auto !important; + } + +/* wide table scroll-bar */ + + table.scrollwide { + display: block; + width: 700px; + background-color: #E0; + overflow: scroll; !important + } + table.scrollwide td { + white-space: nowrap; + } + +/* override navigation sidebar out-of-view on page scrolling */ + + .wy-nav-side { + position: fixed; + padding-bottom: 2em; + width: 300px; + overflow-x: hidden; + overflow-y: scroll; + min-height: 100%; + background: #343131; + z-index: 200; + } + +/* changed side navigation bg colors */ + .wy-side-nav-search { + background: #222c32 !important; + } + + .wy-nav-side { + background: #222c32; + } + + .rst-versions{ + border-top:solid 10px #222c32; + } \ No newline at end of file From bcd748083e4d1a7ecd570059418dc8fd2e8653a6 Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Wed, 2 Jun 2021 13:54:56 -0700 Subject: [PATCH 5/7] Completed sections from GitHub Repo readme --- docs/source/installation.rst | 109 +++++++++++++++++++++++++++++++++++ docs/source/overview.rst | 9 +++ 2 files changed, 118 insertions(+) create mode 100644 docs/source/installation.rst create mode 100644 docs/source/overview.rst diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 000000000..6ef473f5e --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,109 @@ +================ +Installation +================ + +The following instructions are for installing PDC on Linux and Cray machines. GCC version 7 or newer and a version of MPI are needed to install PDC. + +Current PDC tests have been verified with MPICH. To install MPICH, follow the documentation in https://www.mpich.org/static/downloads/3.4.1/mpich-3.4.1-installguide.pdf + +PDC also depends on libfabric and Mercury. We provide detailed instructions for installing libfabric, Mercury, and PDC below. Make sure to record the environmental variables (lines that contains the "export" commands). They are needed for running PDC and to use the libraries again. + +--------------------------- +Install libfabric +--------------------------- + +.. code-block:: bash + wget https://github.com/ofiwg/libfabric/archive/v1.11.2.tar.gz + tar xvzf v1.11.2.tar.gz + cd libfabric-1.11.2 + mkdir install + export LIBFABRIC_DIR=$(pwd)/install + ./autogen.sh + ./configure --prefix=$LIBFABRIC_DIR CC=gcc CFLAG="-O2" + make -j8 + make install + export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH" + export PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH" + +--------------------------- +Install Mercury +--------------------------- + +Make sure the ctest passes. PDC may not work without passing all the tests of Mercury. + +Step 2 in the following is not required. It is a stable commit that has been used to test when these these instructions were written. One may skip it to use the current master branch of Mercury. + +0. git clone https://github.com/mercury-hpc/mercury.git +1. cd mercury +2. git checkout e741051fbe6347087171f33119d57c48cb438438 +3. git submodule update --init +4. export MERCURY_DIR=$(pwd)/install +5. mkdir install +6. cd install +7. cmake ../ -DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF +8. make +9. make install +10. ctest +11. export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" +12. export PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" + +--------------------------- +Install PDC +--------------------------- + +One can replace mpicc to other available MPI compilers. For example, on Cori, cc can be used to replace mpicc. ctest contains both sequential and MPI tests for the PDC settings. These can be used to perform regression tests. + +0. git clone https://github.com/hpc-io/pdc.git +1. cd pdc +2. git checkout stable +3. cd src +4. mkdir install +5. cd install +6. export PDC_DIR=$(pwd) +7. cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc +8. make -j8 +9. ctest + +--------------------------- +Environmental Variables +--------------------------- + +During installation, we have set some environmental variables. These variables may disappear after the close the current session ends. We recommend adding the following lines to ~/.bashrc. (One may also execute them manually after logging in). The MERCURY_DIR and LIBFABRIC_DIR variables should be identical to the values that were set during the installation of Mercury and libfabric. The install path is the path containing bin and lib directory, instead of the one containing the source code. + +export PDC_DIR="where/you/installed/your/pdc" +export MERCURY_DIR="where/you/installed/your/mercury" +export LIBFABRIC_DIR="where/you/installed/your/libfabric" +export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" +export PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" + +One can also manage the path with Spack, which is a lot more easier to load and unload these libraries. + +--------------------------- +Running PDC +--------------------------- + +The ctest under PDC install folder runs PDC examples using PDC APIs. PDC needs to run at least two applications. The PDC servers need to be started first. The client programs that send I/O request to servers as Mercury RPCs are started next. + +We provide a convenient function (mpi_text.sh) to start MPI tests. One needs to change the MPI launching function (mpiexec) with the relevant launcher on a system. On Cori at NERSC, the mpiexec argument needs to be changed to srun. On Theta, it is aprun. On Summit, it is jsrun. + +cd $PDC_DIR/bin +./mpi_test.sh ./pdc_init mpiexec 2 4 + +This is test will start 2 processes for PDC servers. The client program ./pdc_init will start 4 processes. Similarly, one can run any of the client examples in ctest. These source code will provide some knowledge of how to use PDC. For more reference, one may check the documentation folder in this repository. + +--------------------------- +PDC on Cori +--------------------------- + +Installation on Cori is not very different from a regular linux machine. Simply replacing all gcc/mpicc with the default cc compiler on Cori would work. Add options -DCMAKE_C_FLAGS="-dynamic" to the cmake line of PDC. Add -DCMAKE_C_FLAGS="-dynamic" -DCMAKE_CXX_FLAGS="-dynamic" at the end of the cmake line for mercury as well. Finally, "-DMPI_RUN_CMD=srun" is needed for ctest command later. In some instances and on some systems, unload darshan before installation may be needed. + +For job allocation on Cori it is recommended to add "--gres=craynetwork:2" to the command: +salloc -C haswell -N 4 -t 01:00:00 -q interactive --gres=craynetwork:2 + +And to launch the PDC server and the client, add "--gres=craynetwork:1" before the executables: + +Run 4 server processes, each on one node in background: +srun -N 4 -n 4 -c 2 --mem=25600 --cpu_bind=cores --gres=craynetwork:1 ./bin/pdc_server.exe & + +Run 64 client processes that concurrently create 1000 objects in total: +srun -N 4 -n 64 -c 2 --mem=25600 --cpu_bind=cores --gres=craynetwork:1 ./bin/create_obj_scale -r 1000 diff --git a/docs/source/overview.rst b/docs/source/overview.rst new file mode 100644 index 000000000..fd878758e --- /dev/null +++ b/docs/source/overview.rst @@ -0,0 +1,9 @@ +================ +Overview +================ + +Proactive Data Containers (PDC) software provides an object-centric API and a runtime system with a set of data object management services. These services allow placing data in the memory and storage hierarchy, performing data movement asynchronously, and providing scalable metadata operations to find data objects. PDC revolutionizes how data is stored and accessed by using object-centric abstractions to represent data that moves in the high-performance computing (HPC) memory and storage subsystems. PDC manages extensive metadata to describe data objects to find desired data efficiently as well as to store information in the data objects. + +PDC API, data types, and developer notes are available in docs/readme.md. + +More information and publications of PDC is available at https://sdm.lbl.gov/pdc From 34190ed27c3b182d0bc2c20d749a2a55239f0278 Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Wed, 2 Jun 2021 14:15:07 -0700 Subject: [PATCH 6/7] Added code blocks for bash commands --- docs/source/index.rst | 7 ++- docs/source/installation.rst | 86 ++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 8c4b8db3d..3b2164a55 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -4,12 +4,15 @@ contain the root `toctree` directive. Test - Kenneth -Welcome to PDC's documentation! +Proactive Data Containers (PDC) =============================== .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: User Guide + + overview + installation diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 6ef473f5e..3327a80b9 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -12,7 +12,8 @@ PDC also depends on libfabric and Mercury. We provide detailed instructions for Install libfabric --------------------------- -.. code-block:: bash +.. code-block:: Bash + wget https://github.com/ofiwg/libfabric/archive/v1.11.2.tar.gz tar xvzf v1.11.2.tar.gz cd libfabric-1.11.2 @@ -33,19 +34,21 @@ Make sure the ctest passes. PDC may not work without passing all the tests of Me Step 2 in the following is not required. It is a stable commit that has been used to test when these these instructions were written. One may skip it to use the current master branch of Mercury. -0. git clone https://github.com/mercury-hpc/mercury.git -1. cd mercury -2. git checkout e741051fbe6347087171f33119d57c48cb438438 -3. git submodule update --init -4. export MERCURY_DIR=$(pwd)/install -5. mkdir install -6. cd install -7. cmake ../ -DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -8. make -9. make install -10. ctest -11. export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" -12. export PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" +.. code-block:: Bash + + git clone https://github.com/mercury-hpc/mercury.git + cd mercury + git checkout e741051fbe6347087171f33119d57c48cb438438 + git submodule update --init + export MERCURY_DIR=$(pwd)/install + mkdir install + cd install + cmake ../ -DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF + make + make install + ctest + export LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH" + export PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" --------------------------- Install PDC @@ -53,16 +56,18 @@ Install PDC One can replace mpicc to other available MPI compilers. For example, on Cori, cc can be used to replace mpicc. ctest contains both sequential and MPI tests for the PDC settings. These can be used to perform regression tests. -0. git clone https://github.com/hpc-io/pdc.git -1. cd pdc -2. git checkout stable -3. cd src -4. mkdir install -5. cd install -6. export PDC_DIR=$(pwd) -7. cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -8. make -j8 -9. ctest +.. code-block:: Bash + + git clone https://github.com/hpc-io/pdc.git + cd pdc + git checkout stable + cd src + mkdir install + cd install + export PDC_DIR=$(pwd) + cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc + make -j8 + ctest --------------------------- Environmental Variables @@ -70,11 +75,13 @@ Environmental Variables During installation, we have set some environmental variables. These variables may disappear after the close the current session ends. We recommend adding the following lines to ~/.bashrc. (One may also execute them manually after logging in). The MERCURY_DIR and LIBFABRIC_DIR variables should be identical to the values that were set during the installation of Mercury and libfabric. The install path is the path containing bin and lib directory, instead of the one containing the source code. -export PDC_DIR="where/you/installed/your/pdc" -export MERCURY_DIR="where/you/installed/your/mercury" -export LIBFABRIC_DIR="where/you/installed/your/libfabric" -export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" -export PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" +.. code-block:: Bash + + export PDC_DIR="where/you/installed/your/pdc" + export MERCURY_DIR="where/you/installed/your/mercury" + export LIBFABRIC_DIR="where/you/installed/your/libfabric" + export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH" + export PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH" One can also manage the path with Spack, which is a lot more easier to load and unload these libraries. @@ -86,8 +93,10 @@ The ctest under PDC install folder runs PDC examples using PDC APIs. PDC needs t We provide a convenient function (mpi_text.sh) to start MPI tests. One needs to change the MPI launching function (mpiexec) with the relevant launcher on a system. On Cori at NERSC, the mpiexec argument needs to be changed to srun. On Theta, it is aprun. On Summit, it is jsrun. -cd $PDC_DIR/bin -./mpi_test.sh ./pdc_init mpiexec 2 4 +.. code-block:: Bash + + cd $PDC_DIR/bin + ./mpi_test.sh ./pdc_init mpiexec 2 4 This is test will start 2 processes for PDC servers. The client program ./pdc_init will start 4 processes. Similarly, one can run any of the client examples in ctest. These source code will provide some knowledge of how to use PDC. For more reference, one may check the documentation folder in this repository. @@ -98,12 +107,21 @@ PDC on Cori Installation on Cori is not very different from a regular linux machine. Simply replacing all gcc/mpicc with the default cc compiler on Cori would work. Add options -DCMAKE_C_FLAGS="-dynamic" to the cmake line of PDC. Add -DCMAKE_C_FLAGS="-dynamic" -DCMAKE_CXX_FLAGS="-dynamic" at the end of the cmake line for mercury as well. Finally, "-DMPI_RUN_CMD=srun" is needed for ctest command later. In some instances and on some systems, unload darshan before installation may be needed. For job allocation on Cori it is recommended to add "--gres=craynetwork:2" to the command: -salloc -C haswell -N 4 -t 01:00:00 -q interactive --gres=craynetwork:2 + +.. code-block:: Bash + + salloc -C haswell -N 4 -t 01:00:00 -q interactive --gres=craynetwork:2 And to launch the PDC server and the client, add "--gres=craynetwork:1" before the executables: Run 4 server processes, each on one node in background: -srun -N 4 -n 4 -c 2 --mem=25600 --cpu_bind=cores --gres=craynetwork:1 ./bin/pdc_server.exe & + +.. code-block:: Bash + + srun -N 4 -n 4 -c 2 --mem=25600 --cpu_bind=cores --gres=craynetwork:1 ./bin/pdc_server.exe & Run 64 client processes that concurrently create 1000 objects in total: -srun -N 4 -n 64 -c 2 --mem=25600 --cpu_bind=cores --gres=craynetwork:1 ./bin/create_obj_scale -r 1000 + +.. code-block:: Bash + + srun -N 4 -n 64 -c 2 --mem=25600 --cpu_bind=cores --gres=craynetwork:1 ./bin/create_obj_scale -r 1000 From e6206a46587b41f7b3f3033694f7971a4c31a824 Mon Sep 17 00:00:00 2001 From: Kenneth Casimiro Date: Wed, 2 Jun 2021 14:34:48 -0700 Subject: [PATCH 7/7] Added attention box --- docs/source/installation.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 3327a80b9..416daadbc 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -6,7 +6,10 @@ The following instructions are for installing PDC on Linux and Cray machines. GC Current PDC tests have been verified with MPICH. To install MPICH, follow the documentation in https://www.mpich.org/static/downloads/3.4.1/mpich-3.4.1-installguide.pdf -PDC also depends on libfabric and Mercury. We provide detailed instructions for installing libfabric, Mercury, and PDC below. Make sure to record the environmental variables (lines that contains the "export" commands). They are needed for running PDC and to use the libraries again. +PDC also depends on libfabric and Mercury. We provide detailed instructions for installing libfabric, Mercury, and PDC below. + +.. attention:: + Make sure to record the environmental variables (lines that contains the "export" commands). They are needed for running PDC and to use the libraries again. --------------------------- Install libfabric @@ -30,7 +33,8 @@ Install libfabric Install Mercury --------------------------- -Make sure the ctest passes. PDC may not work without passing all the tests of Mercury. +.. attention:: + Make sure the ctest passes. PDC may not work without passing all the tests of Mercury. Step 2 in the following is not required. It is a stable commit that has been used to test when these these instructions were written. One may skip it to use the current master branch of Mercury.