diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f19dff18e..47441a37b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,9 @@ +# Source code +* @kiritigowda @rrawther # Documentation files -docs/* @saadrahim @LisaDelaney @kiritigowda @rrawther -*.md @saadrahim @LisaDelaney @kiritigowda @rrawther -*.rst @saadrahim @LisaDelaney +docs/ @ROCm/rocm-documentation @kiritigowda @rrawther +*.md @ROCm/rocm-documentation @kiritigowda @rrawther +*.rst @ROCm/rocm-documentation @kiritigowda @rrawther +.readthedocs.yaml @ROCm/rocm-documentation # Header directory -library/include/* @saadrahim @LisaDelaney @kiritigowda @rrawther -# Source code -@kiritigowda @rrawther +library/include/ @ROCm/rocm-documentation @kiritigowda @rrawther diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index f0227bec8..d0056aeaf 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -9,4 +9,4 @@ on: jobs: call-workflow-passing-data: name: Documentation - uses: RadeonOpenCompute/rocm-docs-core/.github/workflows/linting.yml@develop + uses: ROCm/rocm-docs-core/.github/workflows/linting.yml@develop diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 19044ecb6..56a904e1a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,8 +13,8 @@ python: - requirements: docs/sphinx/requirements.txt build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.8" + python: "3.10" apt_packages: - "doxygen" diff --git a/docs/README.md b/docs/README.md index 21a599865..1b9abccf2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,3 @@ -

- # rocAL User Guide Today’s deep learning applications require loading and pre-processing data efficiently to achieve high processing throughput. This requires creating efficient processing pipelines fully utilizing the underlying hardware capabilities. Some examples are load and decode data, do a variety of augmentations, color-format conversions, etc. @@ -9,12 +7,12 @@ AMD ROCm Augmentation Library (rocAL) is designed to efficiently do such process These pipelines are programmable by the user using both C++ and Python APIs. ## User Guide Chapters -- [Chapter 1 - Overview](user_guide/ch1.md) -- [Chapter 2 - Architecture Components](user_guide/ch2.md) -- [Chapter 3 - Installation](user_guide/ch3.md) -- [Chapter 4 - Using with Python API](user_guide/ch4.md) -- [Chapter 5 - Framework Integration](user_guide/ch5.md) -- [Chapter 6 - Using with C++ API](user_guide/ch6.md) +* [Chapter 1 - Overview](user_guide/ch1.md) +* [Chapter 2 - Architecture Components](user_guide/ch2.md) +* [Chapter 3 - Installation](user_guide/ch3.md) +* [Chapter 4 - Using with Python API](user_guide/ch4.md) +* [Chapter 5 - Framework Integration](user_guide/ch5.md) +* [Chapter 6 - Using with C++ API](user_guide/ch6.md) ## Key Components of rocAL @@ -88,7 +86,7 @@ amd.rocal.types are enums exported from C++ API to python. Some examples include * Build and install RPP * Build and install MIVisionX which installs rocAL c++ lib -* Go to [rocal_pybind](https://github.com/ROCm/rocAL/tree/master/rocAL_pybind) folder +* Go to the [rocal_pybind](https://github.com/ROCm/rocAL/tree/develop/rocAL_pybind) folder * sudo ./run.sh ### Steps to run MLPerf Resnet50 classification training with rocAL on a system with MI50+ and ROCm @@ -97,12 +95,15 @@ amd.rocal.types are enums exported from C++ API to python. Some examples include * Step 2: Build [MIVisionX Pytorch docker](https://github.com/ROCm/rocAL/blob/master/docker/README.md) * Step 3: Install rocAL python_pybind plugin as described above * Step 4: Clone [MLPerf](https://github.com/rrawther/MLPerf-mGPU) branch and checkout mlperf-v1.1-rocal branch -``` + +``` bash git clone -b mlperf-v1.1-rocal https://github.com/rrawther/MLPerf-mGPU ``` + * Step 5: Modify RN50_AMP_LARS_8GPUS_NCHW.sh or RN50_AMP_LARS_8GPUS_NHWC.sh to reflect correct path for imagenet directory * Step 8: Run RN50_AMP_LARS_8GPUS_NCHC.sh or RN50_AMP_LARS_8GPUS_NHWC.sh -``` + +``` bash ./RN50_AMP_LARS_8GPUS_NCHW.sh (or) ./RN50_AMP_LARS_8GPUS_NHWC.sh @@ -112,9 +113,11 @@ git clone -b mlperf-v1.1-rocal https://github.com/rrawther/MLPerf-mGPU * Refer to the [docker](https://github.com/ROCm/MIVisionX#docker) page for prerequisites and information on building the docker * Step 1: Run the docker image* -```` + +``` bash sudo docker run -it -v :/data -v /:/dockerx -w /dockerx --privileged --device=/dev/kfd --device=/dev/dri --group-add video --shm-size=4g --ipc="host" --network=host -```` +``` + * Optional: Map localhost directory on the docker image * option to map the localhost directory with imagenet dataset folder to be accessed on the docker image. * usage: -v {LOCAL_HOST_DIRECTORY_PATH}:{DOCKER_DIRECTORY_PATH} diff --git a/docs/how-to/framework.rst b/docs/how-to/framework.rst index 9dc4923d8..b99dfcf5d 100644 --- a/docs/how-to/framework.rst +++ b/docs/how-to/framework.rst @@ -75,9 +75,9 @@ Follow these steps: .. code-block:: python :caption: Import libraries for PyTorch - import torch.nn as nn - import torch.nn.functional as F - import torch.optim as optim + import torch.nn as nn + import torch.nn.functional as F + import torch.optim as optim 4. Call the training pipeline with rocAL classification data `loader `_. @@ -85,12 +85,12 @@ Follow these steps: .. code-block:: python :caption: Call the training pipeline - Def get_pytorch_train_loader(self): - print(“in get_pytorch_train_loader function”) - pipe_train = trainPipeline(self.data_path, self.batch_size, self.num_classes, self.one_hot, self.local_rank, - self.world_size, self.num_thread, self.crop, self.rocal_cpu, self.fp16) - pipe_train.build() - train_loader = ROCALClassificationIterator(pipe_train, device=”cpu” if self.rocal_cpu else “cuda”, device_id = self.local_rank) + Def get_pytorch_train_loader(self): + print(“in get_pytorch_train_loader function”) + pipe_train = trainPipeline(self.data_path, self.batch_size, self.num_classes, self.one_hot, self.local_rank, + self.world_size, self.num_thread, self.crop, self.rocal_cpu, self.fp16) + pipe_train.build() + train_loader = ROCALClassificationIterator(pipe_train, device=”cpu” if self.rocal_cpu else “cuda”, device_id = self.local_rank) 5. Run the `training script `_. @@ -130,7 +130,7 @@ Create Data-loading Pipeline Follow these steps: -1. Import libraries for `rocAL_pybind `_. +1. Import libraries for `rocAL_pybind `_. .. code-block:: python :caption: Import libraries @@ -141,7 +141,7 @@ Follow these steps: import amd.rocal.types as types -2. See a rocAL pipeline for TensorFlow below. It reads data from the TFRecords using TFRecord Reader and uses ``fn.decoders.image`` to decode the raw `images `_. +2. See a rocAL pipeline for TensorFlow below. It reads data from the TFRecords using TFRecord Reader and uses ``fn.decoders.image`` to decode the raw `images `_. .. code-block:: python :caption: Pipeline for TensorFlow @@ -170,7 +170,7 @@ Follow these steps: trainPipe.build() -3. Import libraries for `TensorFlow `_. +3. Import libraries for `TensorFlow `_. .. code-block:: python :caption: Import libraries for TensorFlow @@ -193,8 +193,7 @@ Follow these steps: 4. To see and run a sample training script, refer to `rocAL TensorFlow example `_. - -.. _ml-perf: +.. __resnet50: Run MLPerf Resnet50 classification training with rocAL ======================================================= @@ -216,8 +215,7 @@ Run MLPerf Resnet50 classification training with rocAL * Option to map the localhost directory with imagenet dataset folder to be accessed on the docker image. * Usage: ``-v {LOCAL_HOST_DIRECTORY_PATH}:{DOCKER_DIRECTORY_PATH}`` -#. Install rocAL ``python_pybind`` plugin as described above -#. Clone `MLPerf `_ repo and checkout ``mlperf-v1.1-rocal`` branch +#. To see and run a sample training script, refer to `rocAL Imagenet example `_. .. code-block:: shell diff --git a/docs/how-to/using-with-python.rst b/docs/how-to/using-with-python.rst index 442674518..9252f1316 100644 --- a/docs/how-to/using-with-python.rst +++ b/docs/how-to/using-with-python.rst @@ -91,11 +91,10 @@ Given below is an example of a file reader, which takes a folder of images as in images = fn.decoders.image(jpegs, file_root=data_path, device=decoder_device, output_type=types.RGB, shard_id=0, num_shards=1, random_shuffle=True) images = fn.resize(images, device=rocal_device, resize_x=300, resize_y=300) - Defining the Pipeline ------------------------ -To define a pipeline, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/amd/rocal/pipeline.py#L29`. +To define a pipeline, see ``__. .. code-block:: shell :caption: Pipeline Class @@ -192,7 +191,7 @@ Running the Pipeline To run/use the pipeline, simply create a data loader using the pipeline and iterate through it to get the next batch of images with labels. -To run the pipeline, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L168` +To run the pipeline, see ``__. .. code-block:: python :caption: Run the Pipeline @@ -220,7 +219,7 @@ Performing Augmentations rocAL not only reads images from the disk and batches them into tensors, it can also perform various augmentations on those images. -To read images, decode them, and rotate them in the pipeline, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L77` +To read images, decode them, and rotate them in the pipeline, see ``__ .. code-block:: python :caption: Perform Augmentations @@ -246,11 +245,10 @@ To run the pipeline, see: images, labels = pipe_out show_images(images) - rocAL Data Types ========================= -All the rocAL data types are defined under `amd.rocal.types `_. Import this library in the application to access the various data types such as rocAL status, processing mode, tensor output type, image size evaluation policy, image color, tensor layout, decode device, resize scaling mode, and resize interpolation type. +All the rocAL data types are defined under `amd.rocal.types `_. Import this library in the application to access the various data types such as rocAL status, processing mode, tensor output type, image size evaluation policy, image color, tensor layout, decode device, resize scaling mode, and resize interpolation type. Here are some of the commonly used rocAL data types: @@ -269,7 +267,7 @@ Here are some of the commonly used rocAL data types: * tensor_dtype = types.FLOAT * tensor_dtype = types.FLOAT16 -To see the usage of the above-mentioned data types, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/amd/rocal/pipeline.py#L97` +To see the usage of the above-mentioned data types, see ``__. .. code-block:: python diff --git a/docs/index.rst b/docs/index.rst index 8d83db89f..499d91353 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,7 +29,7 @@ The rocAL documentation is structured as follows: * :ref:`install` - .. grid-item-card:: How-to + .. grid-item-card:: How to * :ref:`overview` * :ref:`architecture` @@ -54,7 +54,7 @@ The rocAL documentation is structured as follows: * :ref:`examples` -To contribute to the documentation refer to `Contributing to ROCm `_. +To contribute to the documentation refer to `Contributing to ROCm Docs `_. You can find licensing information on the `Licensing `_ page. diff --git a/docs/install/install.rst b/docs/install/install.rst index 7e6dca28d..920a0640f 100644 --- a/docs/install/install.rst +++ b/docs/install/install.rst @@ -18,21 +18,44 @@ Prerequisites - Ubuntu 20.04 or 22.04 - CentOS 7 - RedHat 8 or 9 - - SLES 15-SP4 - -* `ROCm supported hardware `_ -* Install ROCm with `amdgpu-install `_ with ``--usecase=graphics,rocm --no-32`` -* `RPP `_ -* `AMD OpenVX™ `_ and AMD OpenVX™ Extensions: ``VX_RPP`` and ``AMD Media`` - MIVisionX Components -* `Turbo JPEG `_ - Version 2.0.6.2 from ``https://github.com/rrawther/libjpeg-turbo.git`` -* `Half-precision floating-point `_ library - Version 1.12.0 or higher -* `Google Protobuf `_ - Version 3.12.4 or higher -* `LMBD Library `_ -* `RapidJSON `_ -* `PyBind11 `_ -* `HIP `_ -* OpenMP -* C++17 + - SLES 15-SP5 + +* ROCm-supported hardware + +* Install ROCm `6.1.0` or later with amdgpu-install: Required usecase - rocm + +* HIP + +* RPP + +* MIVisionX + +* rocDecode + +* Half-precision floating-point library - Version `1.12.0` or higher + +* Google Protobuf + +* LMBD Library + +* Python3 and Python3 PIP + +* Python Wheel + +* PyBind11 + +* Turbo JPEG + +* RapidJSON + +* **Optional**: FFMPEG + +* **Optional**: OpenCV + +IMPORTANT +* Compiler features required + * OpenMP + * C++17 Installation instructions ================================ @@ -40,7 +63,7 @@ Installation instructions The installation process uses the following steps: * `ROCm supported hardware install `_ -* Install ROCm with `amdgpu-install `_ with ``--usecase=graphics,rocm --no-32`` +* Install ROCm with `amdgpu-install `_ with ``--usecase=rocm`` * Use either :ref:`package-install` or :ref:`source-install` as described below. .. _package-install: @@ -118,10 +141,10 @@ Prerequisites: - Ubuntu 20.04 or 22.04 - CentOS 7 - RedHat 8 or 9 - - SLES 15-SP4 + - SLES 15-SP5 * `ROCm supported hardware `_ - * Install ROCm with `amdgpu-install `_ with ``--usecase=graphics,rocm --no-32`` + * Install ROCm with `amdgpu-install `_ with ``--usecase=rocm`` Using ``rocAL-setup.py`` script: @@ -129,7 +152,6 @@ Using ``rocAL-setup.py`` script: python rocAL-setup.py --directory [setup directory - optional (default:~/)] --opencv [OpenCV Version - optional (default:4.6.0)] - --protobuf [ProtoBuf Version - optional (default:3.12.4)] --pybind11 [PyBind11 Version - optional (default:v2.10.4)] --reinstall [Remove previous setup and reinstall (default:OFF)[options:ON/OFF]] --backend [rocAL Dependency Backend - optional (default:HIP) [options:OCL/HIP]] @@ -205,4 +227,3 @@ Test package will install ctest module to test rocAL. Follow below steps to test mkdir rocAL-test && cd rocAL-test cmake /opt/rocm/share/rocal/test/ ctest -VV - diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 313992e13..af228f647 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -7,7 +7,7 @@ subtrees: - file: install/install.rst title: Installation - file: how-to/index.rst - title: How To + title: How to subtrees: - entries: - file: how-to/overview.rst diff --git a/docs/sphinx/requirements.in b/docs/sphinx/requirements.in index 316e7cf4d..b1eb21341 100644 --- a/docs/sphinx/requirements.in +++ b/docs/sphinx/requirements.in @@ -1 +1 @@ -rocm-docs-core[api_reference]>=0.24.0 +rocm-docs-core[api_reference]==1.4.1 diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt index 84a952b2a..b0d156078 100644 --- a/docs/sphinx/requirements.txt +++ b/docs/sphinx/requirements.txt @@ -1,124 +1,126 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile requirements.in # -accessible-pygments==0.0.3 +accessible-pygments==0.0.5 # via pydata-sphinx-theme -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -babel==2.12.1 +babel==2.15.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.11.2 +beautifulsoup4==4.12.3 # via pydata-sphinx-theme -breathe==4.34.0 +breathe==4.35.0 # via rocm-docs-core -certifi==2022.12.7 +certifi==2024.2.2 # via requests -cffi==1.15.1 +cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.1.0 +charset-normalizer==3.3.2 # via requests -click==8.1.3 +click==8.1.7 # via # click-log # doxysphinx # sphinx-external-toc click-log==0.4.0 # via doxysphinx -cryptography==40.0.2 +cryptography==42.0.7 # via pyjwt -deprecated==1.2.13 +deprecated==1.2.14 # via pygithub -docutils==0.19 +docutils==0.21.2 # via # breathe # myst-parser # pydata-sphinx-theme # sphinx -doxysphinx==3.3.4 +doxysphinx==3.3.8 # via rocm-docs-core -fastjsonschema==2.16.3 +fastjsonschema==2.19.1 # via rocm-docs-core -gitdb==4.0.10 +gitdb==4.0.11 # via gitpython -gitpython==3.1.31 +gitpython==3.1.43 # via rocm-docs-core -idna==3.4 +idna==3.7 # via requests imagesize==1.4.1 # via sphinx -jinja2==3.1.2 +jinja2==3.1.4 # via # myst-parser # sphinx libsass==0.22.0 # via doxysphinx -lxml==4.9.3 +lxml==4.9.4 # via doxysphinx -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser -markupsafe==2.1.2 +markupsafe==2.1.5 # via jinja2 -mdit-py-plugins==0.3.5 +mdit-py-plugins==0.4.1 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mpire==2.7.1 +mpire==2.10.2 # via doxysphinx -myst-parser==1.0.0 +myst-parser==3.0.1 # via rocm-docs-core -packaging==23.0 +numpy==1.26.4 + # via doxysphinx +packaging==24.0 # via # pydata-sphinx-theme # sphinx -pycparser==2.21 +pycparser==2.22 # via cffi -pydata-sphinx-theme==0.13.3 +pydata-sphinx-theme==0.15.3 # via # rocm-docs-core # sphinx-book-theme -pygithub==1.58.1 +pygithub==2.3.0 # via rocm-docs-core -pygments==2.14.0 +pygments==2.18.0 # via # accessible-pygments # mpire # pydata-sphinx-theme # sphinx -pyjson5==1.6.4 +pyjson5==1.6.6 # via doxysphinx -pyjwt[crypto]==2.6.0 +pyjwt[crypto]==2.8.0 # via pygithub pynacl==1.5.0 # via pygithub -pyparsing==3.1.1 +pyparsing==3.1.2 # via doxysphinx -pyyaml==6.0 +pyyaml==6.0.1 # via # myst-parser # rocm-docs-core # sphinx-external-toc -requests==2.28.2 +requests==2.32.2 # via # pygithub # sphinx -rocm-docs-core[api_reference]>=0.24.0 +rocm-docs-core[api-reference]==1.4.1 # via -r requirements.in -smmap==5.0.0 +smmap==5.0.1 # via gitdb snowballstemmer==2.2.0 # via sphinx -soupsieve==2.4 +soupsieve==2.5 # via beautifulsoup4 -sphinx==5.3.0 +sphinx==7.3.7 # via # breathe # myst-parser @@ -129,33 +131,39 @@ sphinx==5.3.0 # sphinx-design # sphinx-external-toc # sphinx-notfound-page -sphinx-book-theme==1.0.1 +sphinx-book-theme==1.1.2 # via rocm-docs-core -sphinx-copybutton==0.5.1 +sphinx-copybutton==0.5.2 # via rocm-docs-core -sphinx-design==0.4.1 +sphinx-design==0.6.0 # via rocm-docs-core -sphinx-external-toc==0.3.1 +sphinx-external-toc==1.0.1 # via rocm-docs-core -sphinx-notfound-page==0.8.3 +sphinx-notfound-page==1.0.2 # via rocm-docs-core -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +tomli==2.0.1 # via sphinx -tqdm==4.65.2 +tqdm==4.66.4 # via mpire -typing-extensions==4.5.0 - # via pydata-sphinx-theme -urllib3==1.26.15 - # via requests -wrapt==1.15.0 +typing-extensions==4.12.0 + # via + # pydata-sphinx-theme + # pygithub +urllib3==2.2.1 + # via + # pygithub + # requests +wrapt==1.16.0 # via deprecated