Add Purpose, Scope, Roadmap, and a Python specification (#93)

* Add purpose, scope, roadmap, c api, and python spec

* Build docs in build/latest for versioning

* Remove a duplicate warning in Python spec

.github/workflows/deploy.yml

@@ -35,5 +35,5 @@ jobs:
         git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
         git config --global user.email "dlpack-gh-actions-bot@nomail"
         git config --global user.name "dlpack-gh-actions-bot"
-        ghp-import -m "Generate DLPack website" -b gh-pages docs/build/html
+        ghp-import -m "Generate DLPack website" -b gh-pages docs/build
         git push origin gh-pages -f

Makefile

@@ -16,7 +16,7 @@ doc:
 	$(MAKE) -C docs html
 
 show_docs:
-	@python3 -m http.server --directory docs/build/html
+	@python3 -m http.server --directory docs/build/latest
 
 lint:
 	./tests/scripts/task_lint.sh

docs/CMakeLists.txt

@@ -23,13 +23,13 @@ find_program(SPHINX_EXECUTABLE
     DOC "Path to sphinx-build executable")
 
 set(sphinx_source ${CMAKE_CURRENT_SOURCE_DIR}/source)
-set(sphinx_build ${CMAKE_CURRENT_BINARY_DIR}/build)
+set(sphinx_build ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
 set(doxygen_xml_builddir ${doxygen_output_dir}/xml)
 
 add_custom_target(Sphinx ALL
     COMMAND ${SPHINX_EXECUTABLE} -b html
     -Dbreathe_projects.dlpack=${doxygen_xml_builddir}
-    ${sphinx_source} ${sphinx_build}
+    ${sphinx_source} ${sphinx_build} -WT --keep-going
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     COMMENT "Generating documentation with Sphinx"
     VERBATIM)

docs/Makefile

@@ -3,10 +3,10 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?= -Dbreathe_projects.dlpack=../doxygen/xml
+SPHINXOPTS    ?= -Dbreathe_projects.dlpack=../doxygen/xml -WT --keep-going
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
-BUILDDIR      = build
+BUILDDIR      = build/latest
 
 # Put it first so that "make" without argument is like "make help".
 help:
@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -b $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -8,8 +8,8 @@ if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
 set SOURCEDIR=source
-set SPHINXOPTS=-Dbreathe_projects.dlpack=../doxygen/xml
-set BUILDDIR=build
+set SPHINXOPTS=-Dbreathe_projects.dlpack=../doxygen/xml -WT --keep-going
+set BUILDDIR=build/latest
 
 if "%1" == "" goto help
 
@@ -26,7 +26,7 @@ if errorlevel 9009 (
 	exit /b 1
 )
 
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+%SPHINXBUILD% -b %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 goto end
 
 :help

docs/source/c_api.rst

@@ -0,0 +1,35 @@
+.. _c_api:
+
+C API (``dlpack.h``)
+====================
+
+Macros
+~~~~~~
+
+.. doxygendefine:: DLPACK_EXTERN_C
+
+.. doxygendefine:: DLPACK_VERSION
+
+.. doxygendefine:: DLPACK_DLL
+
+Enumerations
+~~~~~~~~~~~~
+
+.. doxygenenum:: DLDeviceType
+
+.. doxygenenum:: DLDataTypeCode
+
+Structs
+~~~~~~~
+
+.. doxygenstruct:: DLDevice
+   :members:
+
+.. doxygenstruct:: DLDataType
+   :members:
+
+.. doxygenstruct:: DLTensor
+   :members:
+
+.. doxygenstruct:: DLManagedTensor
+   :members:

docs/source/conf.py

@@ -17,20 +17,20 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'dlpack'
+project = 'DLPack'
 copyright = '2022, DLPack contributors'
 author = 'DLPack contributors'
 
 # The full version, including alpha/beta/rc tags
-release = '0.6'
+release = '0.6.0'
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['breathe']
+extensions = ['sphinx.ext.intersphinx', 'breathe']
 
 # Add any paths that contain templates here, relative to this directory.
 # templates_path = ['_templates']
@@ -40,6 +40,26 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = []
 
+intersphinx_mapping = {
+    "array_api": ("https://data-apis.org/array-api/latest", None),
+}
+
+html_use_index = True
+html_domain_indices = True
+
+html_theme_options = {
+    "github_url": "https://github.com/dmlc/dlpack",
+    "use_edit_page_button": True,
+    "show_toc_level": 1,
+}
+
+html_context = {
+    "github_user": "dmlc",
+    "github_repo": "dlpack",
+    "github_version": "main",
+    "doc_path": "docs/source",
+}
+
 
 # -- Breathe configuration ---------------------------------------------------
 
@@ -59,4 +79,4 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ['_static']
+html_static_path = ['_static']

docs/source/index.rst

@@ -1,56 +1,75 @@
-Welcome to dlpack's documentation!
+Welcome to DLPack's documentation!
 ==================================
 
-.. toctree::
-   :maxdepth: 2
-   :caption: Contents:
 
-DLPack is a in-memory data buffer to exchange data between different array/tensor representation libraries.
+Purpose
+~~~~~~~
 
-DLPack consists of a header file that defines an intermedieate protocol to represent data in-memory.
-This protocol can be utilied by other array/tensor libraries to convert from their representation to an intermedieate
-representation and vice versa. This create a bridge between different array libraries and enables the user to easily
-integrate multiple such libraries in their projects.
+In order for an ndarray system to interact with a variety of frameworks, a
+stable in-memory data structure is needed.
 
-Many Python array libraries have functions/utilities to convert to and from a dlpack tensor. Below is an example of how one
-can convert a PyTorch tensor to NumPy array:
+DLPack is one such data structure that allows exchange between major
+frameworks. It is developed with inputs from many deep learning system core
+developers. Highlights include:
 
-.. code-block:: Python
+* Minimum and stable: :ref:`simple header <c_api>`
+* Designed for cross hardware: CPU, CUDA, OpenCL, Vulkan, Metal, VPI, ROCm,
+  WebGPU, Hexagon
+* Already a standard with wide community adoption and support:
 
-    import numpy as np
-    import torch
+  * `CuPy <https://docs.cupy.dev/en/stable/reference/generated/cupy.fromDlpack.html>`_
+  * `PyTorch <https://pytorch.org/docs/stable/dlpack.html>`_
+  * `Tensorflow <https://www.tensorflow.org/api_docs/python/tf/experimental/dlpack/from_dlpack>`_
+  * `MXNet <https://mxnet.apache.org/versions/master/api/python/docs/_modules/mxnet/dlpack.html>`_
 
-    # PyTorch tensor
-    torch_tensor = torch.arange(10, device='cpu')
+* Clean C ABI compatible.
 
-    # Convert the PyTorch tensor to NumPy array
-    np_array = np._from_dlpack(torch_tensor)
+  * Means you can create and access it from any language.
+  * It is also essential for building JIT and AOT compilers to support these
+    data types.
 
 
-C API
-=====
+Scope
+~~~~~
 
-.. doxygendefine:: DLPACK_EXTERN_C
+The main design rationale of DLPack is the minimalism. DLPack drops the
+consideration of allocator, device API and focus on the minimum data
+structure. While still considering the need for cross hardware support
+(e.g. the data field is opaque for platforms that does not support normal
+addressing).
 
-.. doxygendefine:: DLPACK_VERSION
+It also simplifies some of the design to remove legacy issues (e.g. everything
+assumes to be row major, strides can be used to support other case, and avoid
+the complexity to consider more layouts).
 
-.. doxygendefine:: DLPACK_DLL
 
-.. doxygenenum:: DLDeviceType
+Roadmap
+~~~~~~~
 
-.. doxygenstruct:: DLDevice
-   :members:
+* C API that could be exposed as a new Python attribute ``__dlpack_info__``
+  for returning API and ABI versions. (see `#34 <https://github.com/dmlc/dlpack/issues/34>`_,
+  `#72 <https://github.com/dmlc/dlpack/pull/72>`_)
+* Clarify alignment requirements. (see
+  `data-apis/array-api#293 <https://github.com/data-apis/array-api/issues/293>`_,
+  `numpy/numpy#20338 <https://github.com/numpy/numpy/issues/20338>`_,
+  `data-apis/array-api#293 (comment) <https://github.com/data-apis/array-api/issues/293#issuecomment-964434449>`_)
+* Adding support for boolean data type (see `#75 <https://github.com/dmlc/dlpack/issues/75>`_)
+* Adding a read-only flag (ABI break) or making it a hard requirement in the spec that
+  imported arrays should be treated as read-only. (see
+  `data-apis/consortium-feedback#1 (comment) <https://github.com/data-apis/consortium-feedback/issues/1#issuecomment-675857753>`_,
+  `data-apis/array-api#191 <https://github.com/data-apis/array-api/issues/191>`_)
+* Standardize C interface for stream exchange. (see `#74 <https://github.com/dmlc/dlpack/issues/74>`_,
+  `#65 <https://github.com/dmlc/dlpack/issues/65>`_)
 
-.. doxygenenum:: DLDataTypeCode
 
-.. doxygenstruct:: DLDataType
-   :members:
+DLPack Documentation
+~~~~~~~~~~~~~~~~~~~~
 
-.. doxygenstruct:: DLTensor
-   :members:
+.. toctree::
+   :maxdepth: 2
 
-.. doxygenstruct:: DLManagedTensor
-   :members:
+   c_api
+   python_spec
 
 
 Indices and tables