init_sphinx_extension_doc.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Python 2/3 compatibility
from __future__ import print_function

from subprocess import check_call, CalledProcessError
import argparse
import os
import re


#######################################
#  Define bool type for argparse
#######################################
def bool_type(argument):
    """
    Implement conversion of boolean input parameters since
    arparse (or bool, depending on the point of view), do not
    handle bool as a type in an intuitive fashion.

    :param argument: The argument to be parsed to a boolean
    :return: The converted value
    """
    try:
        bool(int(argument))
    except ValueError:
        if argument in ('TRUE', 'true', 'True', 't', 'T'):
            return True
        elif argument in ('FALSE', 'false', 'False', 'f', 'F'):
            return False
        else:
            raise ValueError('Parameter could not be converted to type bool')


#######################################
#  Define formatter for argparse
#######################################
class RawDescriptionDefaultHelpArgParseFormatter(argparse.ArgumentDefaultsHelpFormatter,
                                                 argparse.RawDescriptionHelpFormatter):
    """
    Simple derived formatter class for use with argparse used by the
    cl_analysis_driver class. This formatter combines the default
    argparse.ArgumentDefaultsHelpFormatter and
    argparse.RawDescriptionHelpFormatter
    for formatting arguments and help descriptions.

    """
    pass


#####################################################
#  Create README.md
#####################################################
def get_readme(custom_description, custom_release_notes):
    readme_txt = \
"""
# Getting started

## Generate Documentation

* To generate the HTML version of your documentation run ``make html``.
* The [nwb-docutils](https://pypi.org/project/nwb-docutils/) package must be installed.

## Customize Your Extension Documentation

* **extension description**
  * Edit ``source/%s`` to describe your extension.

* **release notes**
  * Edit ``source/%s`` to document improvements and fixes of your extension.

* **documentation build settings**
  * Edit ``source/conf.py`` to customize your extension documentation configuration.
  * Edit ``source/conf_doc_autogen.py`` to customize the format documentation auto-generation based on
    the YAML specification files.


# Overview

The specification documentation uses Sphinx [http://www.sphinx-doc.org/en/stable/index.html](http://www.sphinx-doc.org/en/stable/index.html)

## Rebuilding All

To rebuild the full documentation in html, latex, and PDF simply run:

```
make fulldoc
```

This is a convenience function that is equivalent to:

```
make allclean
make apidoc
make html
make latexpdf
```

## Generating the format documentation from the format spec

The format documentation is auto-generated from the format specification (YAML) sources via:

```
make apidoc
```

This will invoke the executable:

```
nwb_generate_format_docs
```

The script automatically generates a series of .rst, .png, and .pdf files that are stored in the folder `source/format_auto_docs`. The generated .rst files are included in `source/format.rst` and the png and pdf files are used as figures in the autogenerated docs.

The folder `source/format_auto_docs` is reserved for autogenerated files, i.e., files in the folder should not be added or edited by hand as they will be deleted and rebuilt during the full built of the documentation.

By default the Sphinx configuration is setup to always regenerate the sources whenever the docs are being built (see next section). This behavior can be customized via the `spec_doc_rebuild_always` parameter in `source/conf.py`

## Building a specific document type

To build the documentation, run:

```
make <doctype>
```

where `<doctype>` is, e.g., `latexpdf`, `html`, `singlehtml`, or `man`. For a complete list of supported doc-types, see:

```
make help
```

## Cleaning up

`make clean` cleans up all builds of the documentation located in `_build`.

`make allclean` cleans up all builds of the documentation located in `_build` as well as all autogenerated sources stored in `source/format_auto_docs`.

## Configuration

The build of the documentation can be customized via a broad range of Sphinx options in:

`source/conf_doc_autogen.py`

In addition to standard Sphinx options, there are a number of additional options used to customize the content and structure of the autogenerated documents, e.g.:

* `spec_show_yaml_src` - Boolean indicating whether the YAML sources should be included for the different Neurodata types
* `spec_generate_src_file` - Boolean indicating whether the YAML sources of the neurodata_types should be rendered in a separate section (True) or in the same location as the main documentation
* `spec_show_hierarchy_plots` - Boolean indicating whether we should generate and show figures of the hierarchy defined by the specifications as part of the documentation
* `spec_file_per_type` - Boolean indicating whether we should generate separate .inc reStructuredText for each neurodata_type (True)
or should all text be added to the main file (False)
* `spec_show_subgroups_in_tables` - Should subgroups of the main groups be rendered in the table as well. Usually this is disabled since groups are rendered as separate sections in the text
* `spec_appreviate_main_object_doc_in_tables` - Abbreviate the documentation of the main object for which a table is rendered in the table. This is commonly set to True as doc of the main object is already rendered as the main intro for the section describing the object
* `spec_show_title_for_tables` - Add a title for the table showing the specifications.
* `spec_show_subgroups_in_seperate_table` - Should top-level subgroups be listed in a separate table or as part of the main dataset and attributes table
* `spec_table_depth_char` - Char to be used as prefix to indicate the depth of an object in the specification hierarchy. NOTE: The char used should be supported by LaTeX.
* `spec_add_latex_clearpage_after_ndt_sections` - Add a LaTeX clearpage after each main section describing a neurodata_type. This helps in LaTeX to keep the ordering of figures, tables, and code blocks consistent in particular when the hierarchy_plots are included.
* `spec_resolve_type_inc` - Resolve includes to always show the full list of objects that are part of a type (True) or to show only the parts that are actually new to a current type while only linking to base types (False)

In addition, the location of the input format specification can be customized as follows:

* `spec_input_spec_dir` - Directory where the YAML files for the namespace to be documented are located
* `spec_input_namespace_filename` - Name of the YAML file with the specification of the Namespace to be documented
* `spec_input_default_namespace` - Name of the default namespace in the file

Finally, the name and location of output files can be customized as follows:

* `spec_output_dir` - Directory where the autogenerated files should be stored
* `spec_output_master_filename` - Name of the master .rst file that includes all the autogenerated docs
* `spec_output_doc_filename` - Name of the file where the main documentation goes
* `spec_output_src_filename` - Name of the file where the sources of the format spec go. NOTE: This file is only generated if `spec_generate_src_file` is enabled
* `spec_output_doc_type_hierarchy_filename` - Name of the file containing the type hierarchy. (Included in `spec_output_doc_filename`)

In the regular Sphinx `source/conf.py` file, we can then also set:

* `spec_doc_rebuild_always` - Boolean to define whether to always rebuild the source docs from YAML when doing a regular build of the sources (e.g., via `make html`) even if the folder with the source files already exists
""" % (custom_description, custom_release_notes)
    return readme_txt


#####################################################
#  Create text for file describing RTD theme fixes
#####################################################
def get_theme_overwrites():
    return \
"""/* override table width restrictions */
@media screen and (min-width: 767px) {

   .wy-table-responsive table td {
      /* !important prevents the common CSS stylesheets from overriding
         this as on RTD they are loaded after this stylesheet */
      white-space: normal !important;
   }

   .wy-table-responsive {
      overflow: visible !important;
   }
}
"""


#######################################
#  Create text for the credits.rst file
#######################################
def get_credits_rst():
    credits_rst_text = \
"""*******
Credits
*******

.. note::
    Add the credits for your extension here

Acknowledgments
===============


Authors
=======


*****
Legal
*****

License
=======
"""
    return credits_rst_text


#######################################
#  Create text for the format.rst file
#######################################
def get_format_rst(spec_output_dir, output_master, project):
    heading = ""
    for i in range(len(project)):
        heading += '*'

    format_rst_text = \
"""
.. _%s:

%s
%s
%s

Version |release| |today|

.. .. contents::
""" % (project.replace(' ', '_'), heading, project, heading)

    format_rst_text += \
"""
.. include:: %s/%s
""" % (spec_output_dir.rstrip('/'), output_master)

    return format_rst_text


#######################################
#  Create text for the index.rst file
#######################################
def get_index_rst(project, format_master, custom_description=None, custom_release_notes=None):
    index_rst = \
"""Specification for the %s extension
==================================
""" % project

    if custom_description is not None:
        index_rst += \
"""
.. toctree::
    :numbered:
    :maxdepth: 8
    :caption: Table of Contents

    %s
""" % custom_description.split(".")[0]  # strip .rst extension

    index_rst += \
"""
.. toctree::
    :numbered:
    :maxdepth: 3
    :caption: Extension Specification

    %s
""" % format_master

    index_rst += \
"""
.. toctree::
    :maxdepth: 2
    :caption: History & Legal

    %s
    credits
""" % (custom_release_notes.split(".")[0] if custom_release_notes is not None else "")  # strip .rst extension

    index_rst += \
"""
Indices and tables
==================

* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
"""

    return index_rst


####################################################
#  Define text with custom settings for the conf.py
####################################################
def get_custom_settings(utils_dir,
                        spec_dir='../../core',
                        namespace_filename='nwb.namespace.yaml',
                        default_namespace='core',
                        spec_output_dir="_format_auto_docs",
                        output_master='format_spec_main.inc',
                        output_dec_master='format_spec_doc.inc',
                        output_src_master='format_spec_sources.inc',
                        output_hierarchy_master='format_spec_type_hierarchy.inc',
                        output_clean_if_old_hash=True,
                        output_skip_if_current_hash=False,
                        show_yaml_src=True,
                        show_hierarchy_plots=True,
                        generate_src_files=True,
                        generate_file_per_type=True,
                        subgroups_in_seperate_table=True,
                        abbrv_main_obj_in_table=True,
                        show_table_titles=True,
                        table_depth_char='.',
                        latex_clearpage_after_type=True,
                        resolve_type_inc=False,
                        add_pynwb_settings=True,
                        **kwargs   # These are just consumed but not used. This is just a simple trick to allow us to just hand in the arguments from the command line parser directly without having to specify all the argument by hand
                        ):
    """
    Get the custom configuration settings for Sphinx and the autogeneration of sources from YAML.

    :param utils_dir: Folder where the utils directory is located with the generate_format_docs script

    :return: Tuple of strings with: i) the custom sphinx settings for the conf.py file and ii) the custom autogen settings for the conf_doc_autogen.py file
    """
    # Define the base html and latex settings
    custom_sphinx_settings = \
"""############################################################################
#  CUSTOM CONFIGURATIONS ADDED BY THE NWB TOOL FOR GENERATING FORMAT DOCS
###########################################################################

import sphinx_rtd_theme  # noqa: E402
import textwrap  # noqa: E402

# -- Options for intersphinx  ---------------------------------------------
intersphinx_mapping = {'core': ('https://nwb-schema.readthedocs.io/en/latest/', None)}

# -- Generate sources from YAML---------------------------------------------------
# Always rebuild the source docs from YAML even if the folder with the source files already exists
spec_doc_rebuild_always = True


def run_doc_autogen(_):
    # Execute the autogeneration of Sphinx format docs from the YAML sources
    import sys
    import os
    conf_file_dir = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(conf_file_dir)  # Need so that generate format docs can find the conf_doc_autogen file
    from conf_doc_autogen import spec_output_dir

    if spec_doc_rebuild_always or not os.path.exists(spec_output_dir):
        sys.path.append('./docs')  # needed to enable import of generate_format docs
        from nwb_docutils.generate_format_docs import main as generate_docs
        generate_docs()


def setup(app):
    app.connect('builder-inited', run_doc_autogen)
    app.add_stylesheet("theme_overrides.css")  # overrides for wide tables in RTD theme


# -- Customize sphinx settings
numfig = True
autoclass_content = 'both'
autodoc_docstring_signature = True
autodoc_member_order = 'bysource'
add_function_parentheses = False


# -- HTML sphinx options
html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# LaTeX Sphinx options
latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    'preamble': textwrap.dedent(
        '''
        \\\\setcounter{tocdepth}{3}
        \\\\setcounter{secnumdepth}{6}
        \\\\usepackage{enumitem}
        \\\\setlistdepth{100}
        '''),
}
"""

    # Add the input options ot the settings
    custom_autodoc_settings = \
"""# -*- coding: utf-8 -*-
# Configuration file for generating sources for the format documentation from the YAML specification files

import os

# -- Input options for the specification files to be used -----------------------

# Directory where the YAML files for the namespace to be documented are located
spec_input_spec_dir = %s

# Name of the YAML file with the specification of the Namespace to be documented
spec_input_namespace_filename = %s

# Name of the default namespace in the file
spec_input_default_namespace = %s

""" % ("'%s'" % os.path.relpath(os.path.abspath(spec_dir), os.path.abspath(output_master)),
       "'%s'" % namespace_filename,
       "'%s'" % default_namespace)

    # Add options for customizing the locations of output files
    custom_autodoc_settings += """
# -- Options for customizing the locations of output files

# Directory where the autogenerated files should be stored
spec_output_dir = %s

# Name of the master rst file that includes all the autogenerated docs
spec_output_master_filename = %s

# Name of the file where the main documentation goes
spec_output_doc_filename = %s

# Name of the file where the sources of the format spec go. NOTE: This file is only generated if
# spec_generate_src_file is enabled
spec_output_src_filename = %s

# Name of the file containing the type hierarchy. (Included in spec_output_doc_filename)
spec_output_doc_type_hierarchy_filename = %s

# Clean up the output directory before we build if the git hash is out of date
spec_clean_output_dir_if_old_git_hash = %s

# Do not rebuild the format sources if we have previously build the sources and the git hash matches
spec_skip_doc_autogen_if_current_git_hash = %s

""" % ('os.path.join(os.path.dirname(os.path.abspath(__file__)), "%s")' % spec_output_dir,
       "'%s'" % output_master,
       "'%s'" % output_dec_master,
       "'%s'" % output_src_master,
       "'%s'" % output_hierarchy_master,
       "%s" % output_clean_if_old_hash,
       "%s" % output_skip_if_current_hash)

    # Add custom generator options
    custom_autodoc_settings += \
"""
# -- Options for the generation of the documentation from source ----------------

# Should the YAML sources be included for the different modules
spec_show_yaml_src = %s

# Show figure of the hierarchy of objects defined by the spec
spec_show_hierarchy_plots = %s

# Should the sources of the neurodata_types (YAML) be rendered in a separate section (True) or
# in the same location as the base documentation
spec_generate_src_file = %s

# Should separate .inc reStructuredText files be generated for each neurodata_type (True)
# or should all text be added to the main file
spec_file_per_type = %s

# Should top-level subgroups be listed in a separate table or as part of the main dataset and attributes table
spec_show_subgroups_in_seperate_table = %s

# Abbreviate the documentation of the main object for which a table is rendered in the table.
# This is commonly set to True as doc of the main object is alrready rendered as the main intro for the
# section describing the object
spec_appreviate_main_object_doc_in_tables = %s

# Show a title for the tables
spec_show_title_for_tables = %s

# Char to be used as prefix to indicate the depth of an object in the specification hierarchy
spec_table_depth_char = %s  # '→' '.'

# Add a LaTeX clearpage after each main section describing a neurodata_type. This helps in LaTeX to keep the ordering
# of figures, tables, and code blocks consistent in particular when the hierarchy_plots are included
spec_add_latex_clearpage_after_ndt_sections = %s

# Resolve includes to always show the full list of objects that are part of a type (True)
# or to show only the parts that are actually new to a current type while only linking to base types
spec_resolve_type_inc = %s
""" % (str(show_yaml_src),
       str(show_hierarchy_plots),
       str(generate_src_files),
       str(generate_file_per_type),
       str(subgroups_in_seperate_table),
       str(abbrv_main_obj_in_table),
       str(show_table_titles),
       "'%s'" % str(table_depth_char),
       str(latex_clearpage_after_type),
       str(resolve_type_inc))

    # Add specification class settings specific for NWB
    if add_pynwb_settings:
        custom_autodoc_settings += \
"""
# Default type map to be used. This is the type map where dependent namespaces are stored. In the case of
# NWB this is spec_default_type_map = pynwb.get_type_map()
import pynwb  # noqa: E402
spec_default_type_map = pynwb.get_type_map()

# Default specification classes for groups datasets and namespaces. In the case of NWB these are the NWB-specfic
# spec classes. In the general cases these are the spec classes from HDMF
spec_group_spec_cls = pynwb.spec.NWBGroupSpec
spec_dataset_spec_cls = pynwb.spec.NWBDatasetSpec
spec_namespace_spec_cls = pynwb.spec.NWBNamespace
"""

    # Return the custom settings python script
    return custom_sphinx_settings, custom_autodoc_settings


#######################################
#  Create text for our custom makefile
#######################################
def get_makefile(confdir="$(PWD)/source"):

    return \
"""
# Makefile for Sphinx documentation
#

# You can set these variables from the command line.
SPHINXOPTS     =
SPHINXBUILD    = sphinx-build
SPHINXAPIDOC   = sphinx-apidoc
PAPER          =
BUILDDIR       = build
SRCDIR         = ../src
RSTDIR         = source
CONFDIR        = %s
""" % confdir + \
'''

# Internal variables.
PAPEROPT_a4     = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .

.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext fulldoc allclean

help:
	@echo "To update documentation sources from the format specification please use \`make apidoc'"
	@echo ""
	@echo "To build the documentation please use \`make <target>' where <target> is one of"
	@echo "  fulldoc    to rebuild the apidoc, html, and latexpdf all at once"
	@echo "  html       to make standalone HTML files"
	@echo "  dirhtml    to make HTML files named index.html in directories"
	@echo "  singlehtml to make a single large HTML file"
	@echo "  pickle     to make pickle files"
	@echo "  json       to make JSON files"
	@echo "  htmlhelp   to make HTML files and a HTML help project"
	@echo "  qthelp     to make HTML files and a qthelp project"
	@echo "  devhelp    to make HTML files and a Devhelp project"
	@echo "  epub       to make an epub"
	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
	@echo "  text       to make text files"
	@echo "  man        to make manual pages"
	@echo "  texinfo    to make Texinfo files"
	@echo "  info       to make Texinfo files and run them through makeinfo"
	@echo "  gettext    to make PO message catalogs"
	@echo "  changes    to make an overview of all changed/added/deprecated items"
	@echo "  linkcheck  to check all external links for integrity"
	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
	@echo "  apidoc     to to build RST from source code"
	@echo "  clean      to clean all documents built by Sphinx in _build"
	@echo "  allclean   to clean all autogenerated documents both from Sphinx and apidoc"

allclean:
	-rm -rf $(BUILDDIR)/* $(RSTDIR)/modules.rst
	-rm $(RSTDIR)/_format_auto_docs/*.png
	-rm $(RSTDIR)/_format_auto_docs/*.pdf
	-rm $(RSTDIR)/_format_auto_docs/*.rst
	-rm $(RSTDIR)/_format_auto_docs/*.inc

clean:
	-rm -rf $(BUILDDIR)/* $(RSTDIR)/modules.rst

html:
	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
	@echo
	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."

dirhtml:
	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
	@echo
	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."

singlehtml:
	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
	@echo
	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."

pickle:
	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
	@echo
	@echo "Build finished; now you can process the pickle files."

json:
	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
	@echo
	@echo "Build finished; now you can process the JSON files."

htmlhelp:
	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
	@echo
	@echo "Build finished; now you can run HTML Help Workshop with the" \
	      ".hhp project file in $(BUILDDIR)/htmlhelp."

qthelp:
	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
	@echo
	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/sample.qhcp"
	@echo "To view the help file:"
	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/sample.qhc"

devhelp:
	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
	@echo
	@echo "Build finished."
	@echo "To view the help file:"
	@echo "# mkdir -p $$HOME/.local/share/devhelp/sample"
	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/sample"
	@echo "# devhelp"

epub:
	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
	@echo
	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."

latex:
	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
	@echo
	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
	@echo "Run \`make' in that directory to run these through (pdf)latex" \
	      "(use \`make latexpdf' here to do that automatically)."

latexpdf:
	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
	@echo "Running LaTeX files through pdflatex..."
	$(MAKE) -C $(BUILDDIR)/latex all-pdf
	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."

text:
	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
	@echo
	@echo "Build finished. The text files are in $(BUILDDIR)/text."

man:
	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
	@echo
	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."

texinfo:
	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
	@echo
	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
	@echo "Run \`make' in that directory to run these through makeinfo" \
	      "(use \`make info' here to do that automatically)."

info:
	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
	@echo "Running Texinfo files through makeinfo..."
	make -C $(BUILDDIR)/texinfo info
	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."

gettext:
	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
	@echo
	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."

changes:
	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
	@echo
	@echo "The overview file is in $(BUILDDIR)/changes."

linkcheck:
	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
	@echo
	@echo "Link check complete; look for any errors in the above output " \
	      "or in $(BUILDDIR)/linkcheck/output.txt."

doctest:
	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
	@echo "Testing of doctests in the sources finished, look at the " \
	      "results in $(BUILDDIR)/doctest/output.txt."

apidoc:
	PYTHONPATH=$(CONFDIR):$(PYTHONPATH) nwb_generate_format_docs
	@echo
	@echo "Generate rst source files from NWB spec."

fulldoc:
	$(MAKE) allclean
	@echo
	@echo "Rebuilding apidoc, html, latexpdf"
	$(MAKE) apidoc
	$(MAKE) html
	$(MAKE) latexpdf
'''


##########################################
#  Create the argpars command line parser
##########################################
def define_cl_args():
    """
    Create the argument parser for the script
    :return: argparse.ArgumentParser
    """

    parser = argparse.ArgumentParser(description='Create format specification SPHINX documentation for an NWB extension.',
                                     add_help=True,
                                     epilog="\n Copyright: Lawrence Berkeley National Laboratory: 2017",
                                     formatter_class=RawDescriptionDefaultHelpArgParseFormatter)
    parser.add_argument('--project', dest='project', action='store', type=str, required=True,
                        help='Name of the project')
    parser.add_argument('--author', dest='author', action='store', type=str, required=True,
                        help='Name of the author(s). Enclose in "..." if contains whitespaces')
    parser.add_argument('--version', dest='version', action='store', type=str, required=True,
                        help='Version of the project/docs')
    parser.add_argument('--release', dest='release', action='store', type=str, required=True,
                        help='Release of the project/docs')
    parser.add_argument('--language', dest='language', action='store', type=str, required=False,
                        help='Document language', default='English')
    parser.add_argument('--master', dest='master', action='store', type=str, required=False,
                        help='Master document name', default='index.rst')
    parser.add_argument('--credits_master', dest='credits_master', action='store', type=str, required=False,
                        help='Master document name for the credits documentation', default='credits.rst')
    parser.add_argument('--format_master', dest='format_master', action='store', type=str, required=False,
                        help='Master document name for the format documentation', default='format.rst')
    parser.add_argument('--output', dest='output', action='store', type=str, required=True,
                        help='Project directory for the project')
    parser.add_argument('--spec_dir', dest='spec_dir', action='store', type=str, required=True,
                        help='Directory where the format specification YAML source files are located')
    parser.add_argument('--namespace_filename', dest='namespace_filename', action='store', type=str, required=True,
                        help='Name of the format specification YAML source file with the namespace')
    parser.add_argument('--default_namespace', dest='default_namespace', action='store', type=str, required=True,
                        help='Name of the default namespace to be rendered in the documentation')
    parser.add_argument('--spec_output_dir', dest='spec_output_dir', action='store', type=str, required=False,
                        help='Name of the folder within the documentation sources directory where the autogenerated sources should be stored', default="_format_auto_docs")
    parser.add_argument('--output_master', dest='output_master', action='store', type=str, required=False,
                        help='Name of the master reStructuredText file combining all autogenerated docs.', default="format_spec_main.inc")
    parser.add_argument('--output_dec_master', dest='output_dec_master', action='store', type=str, required=False,
                        help='Name of the master reStructuredText file with all the description docs.', default="format_spec_doc.inc")
    parser.add_argument('--output_src_master', dest='output_src_master', action='store', type=str, required=False,
                        help='Name of the master reStructuredText file with all the YAML source docs.', default="format_spec_sources.inc")
    parser.add_argument('--output_hierarchy_master', dest='output_hierarchy_master', action='store', type=str, required=False,
                        help='Name of the master reStructuredText file with the hierarchy of types.', default="format_spec_type_hierarchy.inc")
    parser.add_argument('--utilsdir', dest='utilsdir', action='store', type=str, required=False,
                        help='Unused option (obsoleted following the creation of nwb-docutils package).', default="OBSOLETE")
    parser.add_argument('--copy_utils', dest='copy_utils', action='store', type=str, required=False, default="OBSOLETE",
                        help="Unused option (obsoleted following the creation of nwb-docutils package).")
    parser.add_argument('--resolve_type_inc', dest='resolve_type_inc', action='store', type=bool_type, required=False, default=False,
                        help="Always resolve type includes to show the full spec of an object including inherited components?")
    parser.add_argument('--latex_clearpage_after_type', dest='latex_clearpage_after_type', action='store', type=bool_type, required=False, default=True,
                        help="Add clearpage command in latex after each neurodata_type to esnure figures always appear in the right section at the cost of adding empty space between sections.")
    parser.add_argument('--table_depth_char', dest='table_depth_char', action='store', type=str, required=False, default='.',
                        help="Char to be used to indent entries in description tables to indicate the depth of the object.")
    parser.add_argument('--abbrv_main_obj_in_table', dest='abbrv_main_obj_in_table', action='store', type=bool_type, required=False, default=True,
                        help="Appreviate the description of the main object in the description tables?")
    parser.add_argument('--subgroups_in_seperate_table', dest='subgroups_in_seperate_table', action='store', type=bool_type, required=False, default=True,
                        help="Render the description of top level subgroups in separate tables rather than as part of the attributes/dataset table?.")
    parser.add_argument('--generate_file_per_type', dest='generate_file_per_type', action='store', type=bool_type, required=False, default=True,
                        help="Generate separate reStructuredText files with .inc ending for each type/neurodata_type rather than rendering all description in a single file?")
    parser.add_argument('--generate_src_file', dest='generate_src_file', action='store', type=bool_type, required=False, default=True,
                        help="Generate separate reStructuredText files for the YAML sources rather than including them in the description files?")
    parser.add_argument('--show_hierarchy_plots', dest='show_hierarchy_plots', action='store', type=bool_type, required=False, default=True,
                        help="Generate and show plots of the hierarchy for each tyep/neurodata_type?")
    parser.add_argument('--show_yaml_src', dest='show_yaml_src', action='store', type=bool_type, required=False, default=True,
                        help="Show the source specification as YAML.")
    parser.add_argument('--custom_description', dest='custom_description', action='store', type=str, required=False, default=None,
                        help="Name of the custom RST file where further text for the description of the extension should be added. If not set then the custom description section will be omitted.")
    parser.add_argument('--external_description', dest='external_description', action='store', type=str, required=False, default=None,
                        help='Path to the rst file that should be included via a reference in the description documentation. Typically this file will be provided alongside with the spec_dir.')
    parser.add_argument('--custom_release_notes', dest='custom_release_notes', action='store', type=str, required=False, default=None,
                        help="Name of the custom RST file where further text for release notes for the extension should be added. If not set then the custom release_notes section will be omitted.")
    parser.add_argument('--external_release_notes', dest='external_release_notes', action='store', type=str, required=False, default=None,
                        help='Path to the rst file that should be included via a reference in the release notes documentation. Typically this file will be provided alongside with the spec_dir.')
    parser.add_argument('--add_pynwb_settings', dest='add_pynwb_settings', action='store', type=bool_type, required=False, default=True,
                        help="Add NWB:N specific settings to use the PyNWB spec classes and type map by to get access to the NWB:N core.")
    return parser


def _read_lines(filepath):
    with open(filepath, 'r') as fp:
        return fp.readlines()


def _write_lines(filepath, lines):
    with open(filepath, 'w', encoding='utf-8') as fp:
        fp.writelines(lines)


def _replace_line_text(filepath, text_to_replace, replace):
    """Update all lines of filepath starting with text"""
    updated_lines = []
    replaced = 0
    for line in _read_lines(filepath):
        if line.startswith(text_to_replace):
            line = replace + line[len(text_to_replace):]
        else:
            replaced = replaced + 1
        updated_lines.append(line)
    _write_lines(filepath, updated_lines)
    return replaced


def _remove_lines(filepath, lines_to_remove):
    """Update filepath removing lines_to_remove.
    """
    updated_lines = []
    removed = 0
    for line in _read_lines(filepath):
        if line.strip() not in lines_to_remove:
            updated_lines.append(line)
        else:
            removed = removed + 1
    _write_lines(filepath, updated_lines)
    return removed


def _remove_lines_after(filepath, regex, number_of_line=1):
    """Update filepath removing N lines after the line matching the provided regex."""
    p = re.compile(regex)
    updated_lines = []
    removed = 0
    lines_to_remove = 0

    for line in _read_lines(filepath):
        if lines_to_remove == 0:
            updated_lines.append(line)
            if p.match(line):
                lines_to_remove = number_of_line
        else:
            lines_to_remove = lines_to_remove - 1
            removed = removed + 1

    _write_lines(filepath, updated_lines)
    return removed


def _append_to_file(filepath, content):
    print("Updating file %s" % filepath)
    with open(filepath, "a", encoding="utf-8") as outfile:
        outfile.write(content)


def _write_file(filepath, content):
    print("Creating file %s" % filepath)
    with open(filepath, "w", encoding="utf-8") as outfile:
        outfile.write(content)


#######################################
#  Run sphinx-quickstart
#######################################
def init_sphinx(project, author, version, release, language, sphinx_master, output):

    command = ['sphinx-quickstart',
               '--quiet',
               '--sep',
               '-p',
               project,
               '-a',
               author,
               '-v',
               version,
               '-r',
               release,
               '-l',
               language,
               '--master',
               sphinx_master,
               '--ext-ifconfig',
               '--ext-autodoc',
               '--ext-intersphinx',
               '--no-makefile',
               output]
    try:
        check_call(command, shell=False)

        conf_py = os.path.join(output, "source", "conf.py")

        print("Cleaning file %s" % conf_py)

        # Remove example of intersphinx configuration
        _remove_lines(
            conf_py,
            ['# Example configuration for intersphinx: refer to the Python standard library.',
             'intersphinx_mapping = {\'https://docs.python.org/\': None}']
        )

        # Fix indent of intersphinx extension list
        _replace_line_text(conf_py, '    \'sphinx.ext.', '              \'sphinx.ext.')

        # Remove timestamp info added by sphinx-quickstart
        _remove_lines_after(conf_py, '# simulation_output documentation build configuration file, created by', 5)
        _remove_lines(conf_py, ['# simulation_output documentation build configuration file, created by'])

        # Remove empty directories
        os.rmdir(os.path.join(output, "build"))
        os.rmdir(os.path.join(output, "source", "_templates"))

        # Remove empty lines from the end of files
        _write_lines(conf_py, _read_lines(conf_py)[:-2])
    except CalledProcessError:
        exit(0)


###########################################
#  Write the custom configuration settings
###########################################
def write_custom_conf(output, **kwargs):
    # Write the custom settings files
    custom_sphinx_settings, custom_doc_autogen_settings = get_custom_settings(output, **kwargs)
    outfilename = os.path.join(output, 'source/conf.py')
    _append_to_file(outfilename, custom_sphinx_settings)

    # Write the custom doc autogen settings
    outfilename = os.path.join(output, 'source/conf_doc_autogen.py')
    _append_to_file(outfilename, custom_doc_autogen_settings)


#######################################
#  Write RTD theme overwrites
#######################################
def write_theme_overwrites(output):
    static_path = os.path.join(output, 'source/_static')
    if not os.path.exists(static_path):
        os.mkdir(static_path)
    outfilename = os.path.join(static_path, 'theme_overrides.css')
    _write_file(outfilename, get_theme_overwrites())


#######################################
#  Write custom makefile
#######################################
def write_makefile(output):
    outfilename = os.path.join(output, 'Makefile')
    _write_file(outfilename, get_makefile())


#######################################
#  Write credits rst
#######################################
def write_credits_rst(output, credits_master):
    outfilename = os.path.join(output, 'source/%s' % credits_master)
    _write_file(outfilename, get_credits_rst())


#######################################
#  Write format rst
#######################################
def write_format_rst(output, format_master, project, spec_output_dir, output_master):
    outfilename = os.path.join(output, 'source/%s' % format_master)
    _write_file(outfilename, get_format_rst(
        spec_output_dir=spec_output_dir,
        output_master=output_master,
        project=project
    ))


######################################
#  Write the custom description file
######################################
def write_custom_description(output, custom_description, external_description):
    outfilename = os.path.join(output, 'source/%s' % custom_description if custom_description is not None else "format_description.rst")
    custom_description_text = None
    if external_description is not None:
        custom_description_text = ".. include:: %s\n" % os.path.join(
            os.path.relpath(os.path.dirname(os.path.abspath(external_description)),
                            start=os.path.dirname(os.path.abspath(outfilename))),
            os.path.basename(external_description))
    elif custom_description is not None:
        custom_description_text = \
"""Overview
========

.. note::
    Add the description of your extension here
"""
    if custom_description_text is not None:
        _write_file(outfilename, custom_description_text)
        return os.path.basename(outfilename)
    else:
        return None


######################################
#  Write the custom description file
######################################
def write_custom_release_notes(output, custom_release_notes, external_release_notes):
    outfilename = os.path.join(output, 'source/%s' % custom_release_notes if custom_release_notes is not None else 'format_release_notes.rst')
    custom_release_notes_text = None
    if external_release_notes is not None:
        custom_release_notes_text = ".. include:: %s\n" % os.path.join(
            os.path.relpath(os.path.dirname(os.path.abspath(external_release_notes)),
                            start=os.path.dirname(os.path.abspath(outfilename))),
            os.path.basename(external_release_notes))
    elif custom_release_notes is not None:
        custom_release_notes_text = \
"""Release Notes
=============

.. note::
    Add the release notes of your extension here
"""
    if custom_release_notes_text is not None:
        _write_file(outfilename, custom_release_notes_text)
        return os.path.basename(outfilename)
    else:
        return None


#########################################################################
#  Write custom index rst and delete the default one created by sphinx
########################################################################
def write_index_rst(output, format_master, project, master, sphinx_master, custom_description=None, custom_release_notes=None):
    os.remove(os.path.join(output, 'source/%s.rst' % sphinx_master))
    outfilename = os.path.join(output, 'source/%s' % master)
    _write_file(outfilename, get_index_rst(
        project=project, format_master=format_master,
        custom_description=custom_description,
        custom_release_notes=custom_release_notes
    ))


#######################################
#  Write the readme file
#######################################
def write_readme(output, custom_description, custom_release_notes):
    outfilename = os.path.join(output, 'README.md')
    _write_file(outfilename, get_readme(
        custom_description=custom_description,
        custom_release_notes=custom_release_notes
    ))


#######################################
#  MAIN
#######################################
def main(args=None):
    parser = define_cl_args()
    clargs = vars(parser.parse_args(args))
    clargs['output'] = os.path.abspath(clargs['output'])
    sphinx_master = 'index'  # This is just a temporary master file that sphinx-quickstart will generate. We'll delete that file and write our own custom index file later
    init_sphinx(project=clargs['project'],
                author=clargs['author'],
                version=clargs['version'],
                release=clargs['release'],
                language=clargs['language'],
                sphinx_master=sphinx_master,
                output=clargs['output'])
    write_theme_overwrites(output=clargs['output'])
    write_custom_conf(**clargs)
    write_makefile(output=clargs['output'])
    custom_description_file = write_custom_description(output=clargs['output'],
                                                       custom_description=clargs['custom_description'],
                                                       external_description=clargs['external_description'])
    custom_release_notes_file = write_custom_release_notes(output=clargs['output'],
                                                           custom_release_notes=clargs['custom_release_notes'],
                                                           external_release_notes=clargs['external_release_notes'])

    write_credits_rst(output=clargs['output'],
                      credits_master=clargs['credits_master'])

    write_format_rst(output=clargs['output'],
                     format_master=clargs['format_master'],
                     project=clargs['project'],
                     spec_output_dir=clargs['spec_output_dir'],
                     output_master=clargs['output_master'])
    write_index_rst(output=clargs['output'],
                    format_master=clargs['format_master'].split('.')[0],  # strip '.rst' extension
                    project=clargs['project'],
                    master=clargs['master'],
                    sphinx_master=sphinx_master,
                     custom_description=custom_description_file,
                     custom_release_notes=custom_release_notes_file)

    write_readme(output=clargs['output'],
                 custom_description=custom_description_file,
                 custom_release_notes=custom_release_notes_file)


if __name__ == "__main__":
    main()


#  python init_sphinx_extension_doc.py --project test --author "Oliver Ruebel" --version 1.2.3 --release alpha --master index.rst --output ../testproject --spec_dir ../../core --namespace_filename nwb.namespace.yaml --default_namespace core --custom_description format_description.rst --custom_release_notes format_release_notes.rst --external_description ../format/source/format_description.inc --external_release_notes ../format/source/format_release_notes.inc