From 31393b1952dd0a78b24a5a518d7a37b5864e98d6 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Tue, 11 Apr 2023 14:10:24 -0400 Subject: [PATCH 1/4] feat: sphinx plugin to get docs outside the main tree --- docs/_ext/helloworld.py | 77 +++++++++++++++++++ docs/conf.py | 7 +- docs/index.rst | 8 +- .../decisions/0001-plugin-contexts.rst | 1 - .../how_tos/how_to_create_a_plugin_app.rst | 1 - .../how_to_enable_plugins_for_an_ida.rst | 1 - docs/plugins/readme.rst | 1 - 7 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 docs/_ext/helloworld.py delete mode 100644 docs/plugins/decisions/0001-plugin-contexts.rst delete mode 100644 docs/plugins/how_tos/how_to_create_a_plugin_app.rst delete mode 100644 docs/plugins/how_tos/how_to_enable_plugins_for_an_ida.rst delete mode 100644 docs/plugins/readme.rst diff --git a/docs/_ext/helloworld.py b/docs/_ext/helloworld.py new file mode 100644 index 00000000..7d102555 --- /dev/null +++ b/docs/_ext/helloworld.py @@ -0,0 +1,77 @@ +from docutils import nodes +import shutil + +from docutils.parsers.rst import Directive +from os import path, walk, symlink, path, mkdir +import sys + +from sphinx.builders import Builder +from sphinx.util import logging + +logger = logging.getLogger(__name__) + +def should_remove_dir(doc_root, repo_root, subdir): + always_remove = subdir in ["build", "dist", ".tox", ".github", ".git", ".idea", '__pycache__'] + egg = subdir.endswith("egg-info") + if always_remove or egg: + return True + is_doc_root = path.abspath(subdir) == path.abspath(doc_root) + return is_doc_root + +def rm_symlinks(app, exception): + srcdir = app.srcdir + symlink_dir = app.config.symlink_sub_dir + shutil.rmtree(path.join(srcdir, symlink_dir)) + +def symlink_a_lot(app, config): + srcdir = app.srcdir + symlink_dir = config.symlink_sub_dir + try: + mkdir(path.join(srcdir, symlink_dir)) + except FileExistsError: + pass + logger.info(f"{srcdir=}") + + for root, dirs, files in walk(config.repo_root, topdown=True): + + if path.commonpath([srcdir, root]) == srcdir: + continue + try: + if not root == config.repo_root: + new_dir = path.join(srcdir, symlink_dir, path.relpath(root, start=config.repo_root)) + logger.info(f"============{root=}================") + logger.info(f"============{new_dir=}=============") + mkdir(new_dir) + except FileExistsError: + pass + [dirs.remove(d) for d in list(dirs) if should_remove_dir(srcdir, config.repo_root, d)] + for afile in files: + if afile.endswith('.rst'): + full_path = path.join(root, afile) + the_symlink = path.join(srcdir, symlink_dir, path.relpath(path.join(root, afile), start=config.repo_root)) + logger.info(f"============== {full_path=}, {the_symlink=} ===========") + try: + symlink(full_path, the_symlink) + except FileExistsError: + pass + + + +def setup(app): + app.add_config_value("repo_root", "", 'html', [str]) + app.add_config_value("symlink_sub_dir", "symlinks", 'html', [str]) + + + app.connect('config-inited', symlink_a_lot) + app.connect('build-finished', rm_symlinks) + + + return { + + 'version': '0.1', + + 'parallel_read_safe': True, + + 'parallel_write_safe': True, + + } diff --git a/docs/conf.py b/docs/conf.py index b654e018..6529af0d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,6 +37,7 @@ def get_version(*file_paths): REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(REPO_ROOT) +sys.path.append("./_ext") VERSION = get_version('../edx_django_utils', '__init__.py') @@ -67,7 +68,8 @@ def get_version(*file_paths): 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.ifconfig', - 'sphinx.ext.napoleon' + 'sphinx.ext.napoleon', + 'helloworld', ] # A list of warning types to suppress arbitrary warning messages. @@ -469,6 +471,9 @@ def get_version(*file_paths): # # epub_use_index = True +repo_root = os.path.abspath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')) +symlink_sub_dir = "symlinks" + # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { diff --git a/docs/index.rst b/docs/index.rst index a57208ab..918784b4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,20 +49,20 @@ EdX utilities for Django Application development. :maxdepth: 1 :caption: Plugins README - plugins/readme + symlinks/edx_django_utils/plugins/README .. toctree:: :maxdepth: 1 :caption: Plugins Decisions - plugins/decisions/0001-plugin-contexts + symlinks/edx_django_utils/plugins/docs/decisions/0001-plugin-contexts .. toctree:: :maxdepth: 1 :caption: Plugins How-Tos - plugins/how_tos/how_to_enable_plugins_for_an_ida - plugins/how_tos/how_to_create_a_plugin_app + symlinks/edx_django_utils/plugins/docs/how_tos/how_to_enable_plugins_for_an_ida + symlinks/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app Indices and tables diff --git a/docs/plugins/decisions/0001-plugin-contexts.rst b/docs/plugins/decisions/0001-plugin-contexts.rst deleted file mode 100644 index 65560f28..00000000 --- a/docs/plugins/decisions/0001-plugin-contexts.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/plugins/docs/decisions/0001-plugin-contexts.rst diff --git a/docs/plugins/how_tos/how_to_create_a_plugin_app.rst b/docs/plugins/how_tos/how_to_create_a_plugin_app.rst deleted file mode 100644 index 8e672449..00000000 --- a/docs/plugins/how_tos/how_to_create_a_plugin_app.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst diff --git a/docs/plugins/how_tos/how_to_enable_plugins_for_an_ida.rst b/docs/plugins/how_tos/how_to_enable_plugins_for_an_ida.rst deleted file mode 100644 index 62091938..00000000 --- a/docs/plugins/how_tos/how_to_enable_plugins_for_an_ida.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/plugins/docs/how_tos/how_to_enable_plugins_for_an_ida.rst diff --git a/docs/plugins/readme.rst b/docs/plugins/readme.rst deleted file mode 100644 index 813dcb6d..00000000 --- a/docs/plugins/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../edx_django_utils/plugins/README.rst From 49c72eefd0f75b50212e14355625971bbd50dccb Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Tue, 11 Apr 2023 14:34:33 -0400 Subject: [PATCH 2/4] fixup!: cleanup --- docs/_ext/helloworld.py | 77 ------------------------ docs/_ext/include_all_files.py | 104 +++++++++++++++++++++++++++++++++ docs/conf.py | 2 +- 3 files changed, 105 insertions(+), 78 deletions(-) delete mode 100644 docs/_ext/helloworld.py create mode 100644 docs/_ext/include_all_files.py diff --git a/docs/_ext/helloworld.py b/docs/_ext/helloworld.py deleted file mode 100644 index 7d102555..00000000 --- a/docs/_ext/helloworld.py +++ /dev/null @@ -1,77 +0,0 @@ -from docutils import nodes -import shutil - -from docutils.parsers.rst import Directive -from os import path, walk, symlink, path, mkdir -import sys - -from sphinx.builders import Builder -from sphinx.util import logging - -logger = logging.getLogger(__name__) - -def should_remove_dir(doc_root, repo_root, subdir): - always_remove = subdir in ["build", "dist", ".tox", ".github", ".git", ".idea", '__pycache__'] - egg = subdir.endswith("egg-info") - if always_remove or egg: - return True - is_doc_root = path.abspath(subdir) == path.abspath(doc_root) - return is_doc_root - -def rm_symlinks(app, exception): - srcdir = app.srcdir - symlink_dir = app.config.symlink_sub_dir - shutil.rmtree(path.join(srcdir, symlink_dir)) - -def symlink_a_lot(app, config): - srcdir = app.srcdir - symlink_dir = config.symlink_sub_dir - try: - mkdir(path.join(srcdir, symlink_dir)) - except FileExistsError: - pass - logger.info(f"{srcdir=}") - - for root, dirs, files in walk(config.repo_root, topdown=True): - - if path.commonpath([srcdir, root]) == srcdir: - continue - try: - if not root == config.repo_root: - new_dir = path.join(srcdir, symlink_dir, path.relpath(root, start=config.repo_root)) - logger.info(f"============{root=}================") - logger.info(f"============{new_dir=}=============") - mkdir(new_dir) - except FileExistsError: - pass - [dirs.remove(d) for d in list(dirs) if should_remove_dir(srcdir, config.repo_root, d)] - for afile in files: - if afile.endswith('.rst'): - full_path = path.join(root, afile) - the_symlink = path.join(srcdir, symlink_dir, path.relpath(path.join(root, afile), start=config.repo_root)) - logger.info(f"============== {full_path=}, {the_symlink=} ===========") - try: - symlink(full_path, the_symlink) - except FileExistsError: - pass - - - -def setup(app): - app.add_config_value("repo_root", "", 'html', [str]) - app.add_config_value("symlink_sub_dir", "symlinks", 'html', [str]) - - - app.connect('config-inited', symlink_a_lot) - app.connect('build-finished', rm_symlinks) - - - return { - - 'version': '0.1', - - 'parallel_read_safe': True, - - 'parallel_write_safe': True, - - } diff --git a/docs/_ext/include_all_files.py b/docs/_ext/include_all_files.py new file mode 100644 index 00000000..19a037f2 --- /dev/null +++ b/docs/_ext/include_all_files.py @@ -0,0 +1,104 @@ +""" +A Sphinx extension to allow the inclusion of documents that are outside of the main docs directory. + +Before building, look in the entire repository (minus the main docs directory) for any .rst files. Symlink these files +inside the main docs directory so they are visible to Sphinx and can be included in index.rst. After the build, removes +the directory containing all the symlinks. + +Requires new config values: +repo_root: the absolute path of the root of the repository +symlink_sub_dir: the name of the subdirectory in which to put the symlinks. Default "symlinks" +""" + +import shutil +from os import mkdir, path, symlink, walk + +from sphinx.util import logging + +logger = logging.getLogger(__name__) + + +def should_remove_dir(subdir): + """ + Determine if we should skip a directory + + Parameters: + subdir - a directory returned from os.walk + + Returns: + True if the directory is only for generated files or should otherwise be ignored + """ + return subdir in ["build", "dist", ".tox", ".github", ".git", ".idea", '__pycache__'] or subdir.endswith("egg-info") + + +def rm_symlinks(app, exception): + """ + Remove the symlink directory + """ + srcdir = app.srcdir + symlink_dir = app.config.symlink_sub_dir + shutil.rmtree(path.join(srcdir, symlink_dir)) + + +def create_symlinks(app, config): + """ + Create symlinks + :param app: + :param config: + :return: + """ + srcdir = app.srcdir + symlink_dir = config.symlink_sub_dir + try: + mkdir(path.join(srcdir, symlink_dir)) + except FileExistsError: + logger.error(f"Cannot create directory for symlinks. Directory {symlink_dir} already exists in {srcdir}") + raise + for root, dirs, files in walk(config.repo_root, topdown=True): + # if we're in a subdirectory of the root docs folder, we can already access all the docs here, so skip + if path.commonpath([srcdir, root]) == srcdir: + continue + + [dirs.remove(d) for d in list(dirs) if should_remove_dir(srcdir, config.repo_root, d)] + + try: + # if necessary, make the new subdir for these symlinks + if not root == config.repo_root: + new_dir = path.join(srcdir, symlink_dir, path.relpath(root, start=config.repo_root)) + mkdir(new_dir) + except FileExistsError: + logger.info(f"Not making new subdir for {root}, subdir already exists") + pass + + for doc_file in files: + # only adds rsts for now + if doc_file.endswith('.rst'): + # full_path will be an absolute path like /path/to/repo/app/docs/mydoc.rst + full_path = path.join(root, doc_file) + + # eg app/docs/mydoc.rst + relative_to_repo_root = path.relpath(full_path, start=config.repo_root) + + # add the relative path to the doc source directory, eg docs/app/docs/mydoc.rst + symlink_path = path.join(srcdir, symlink_dir, relative_to_repo_root) + try: + symlink(full_path, symlink_path) + except FileExistsError: + logger.warn(f"Cannot create {symlink_path}, file already exists") + pass + + +def setup(app): + """ + Entry point into Sphinx + """ + app.add_config_value("repo_root", "", 'html', [str]) + app.add_config_value("symlink_sub_dir", "symlinks", 'html', [str]) + app.connect('config-inited', create_symlinks) + app.connect('build-finished', rm_symlinks) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/conf.py b/docs/conf.py index 6529af0d..21feb10f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -69,7 +69,7 @@ def get_version(*file_paths): 'sphinx.ext.intersphinx', 'sphinx.ext.ifconfig', 'sphinx.ext.napoleon', - 'helloworld', + 'include_all_files', ] # A list of warning types to suppress arbitrary warning messages. From 43a499385cf31634a9df6aac703521d999ae60e2 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Wed, 12 Apr 2023 08:14:04 -0400 Subject: [PATCH 3/4] fixup!: should_rm_dir --- docs/_ext/include_all_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_ext/include_all_files.py b/docs/_ext/include_all_files.py index 19a037f2..5ad19f58 100644 --- a/docs/_ext/include_all_files.py +++ b/docs/_ext/include_all_files.py @@ -59,7 +59,7 @@ def create_symlinks(app, config): if path.commonpath([srcdir, root]) == srcdir: continue - [dirs.remove(d) for d in list(dirs) if should_remove_dir(srcdir, config.repo_root, d)] + [dirs.remove(d) for d in list(dirs) if should_remove_dir(d)] try: # if necessary, make the new subdir for these symlinks From 71e69a66cef112f539c704fb4ba5f4dc931d5696 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Wed, 12 Apr 2023 11:19:34 -0400 Subject: [PATCH 4/4] fixup!: get rid of monitoring stuff --- docs/index.rst | 16 +++++----- docs/monitoring/README.rst | 30 ------------------- .../0001-monitoring-by-code-owner.rst | 1 - .../0002-custom-monitoring-language.rst | 1 - .../0003-code-owner-for-celery-tasks.rst | 1 - .../0004-code-owner-theme-and-squad.rst | 1 - ..._code_owner_custom_attribute_to_an_ida.rst | 1 - .../how_tos/search_new_relic_nrql.rst | 1 - ..._monitoring_for_squad_or_theme_changes.rst | 1 - .../how_tos/using_custom_attributes.rst | 1 - edx_django_utils/plugins/README.rst | 4 +-- 11 files changed, 10 insertions(+), 48 deletions(-) delete mode 100644 docs/monitoring/README.rst delete mode 100644 docs/monitoring/decisions/0001-monitoring-by-code-owner.rst delete mode 100644 docs/monitoring/decisions/0002-custom-monitoring-language.rst delete mode 100644 docs/monitoring/decisions/0003-code-owner-for-celery-tasks.rst delete mode 100644 docs/monitoring/decisions/0004-code-owner-theme-and-squad.rst delete mode 100644 docs/monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida.rst delete mode 100644 docs/monitoring/how_tos/search_new_relic_nrql.rst delete mode 100644 docs/monitoring/how_tos/update_monitoring_for_squad_or_theme_changes.rst delete mode 100644 docs/monitoring/how_tos/using_custom_attributes.rst diff --git a/docs/index.rst b/docs/index.rst index 918784b4..4b8d0453 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,19 +31,19 @@ EdX utilities for Django Application development. :maxdepth: 1 :caption: Monitoring Decisions - monitoring/decisions/0001-monitoring-by-code-owner - monitoring/decisions/0002-custom-monitoring-language - monitoring/decisions/0003-code-owner-for-celery-tasks - monitoring/decisions/0004-code-owner-theme-and-squad + symlinks/edx_django_utils/monitoring/docs/decisions/0001-monitoring-by-code-owner + symlinks/edx_django_utils/monitoring/docs/decisions/0002-custom-monitoring-language + symlinks/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks + symlinks/edx_django_utils/monitoring/docs/decisions/0004-code-owner-theme-and-squad .. toctree:: :maxdepth: 1 :caption: Monitoring How-Tos - monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida - monitoring/how_tos/using_custom_attributes - monitoring/how_tos/search_new_relic_nrql - monitoring/how_tos/update_monitoring_for_squad_or_theme_changes + symlinks/edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida + symlinks/edx_django_utils/monitoring/docs/how_tos/using_custom_attributes + symlinks/edx_django_utils/monitoring/docs/how_tos/search_new_relic_nrql + symlinks/edx_django_utils/monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes .. toctree:: :maxdepth: 1 diff --git a/docs/monitoring/README.rst b/docs/monitoring/README.rst deleted file mode 100644 index 5b41c692..00000000 --- a/docs/monitoring/README.rst +++ /dev/null @@ -1,30 +0,0 @@ -:orphan: - -This README explains the purpose of this directory in the form of a decision document. - -Context -------- - -At this time, we don't have a standard way to pull in rst docs from outside of the main docs directory. See https://github.com/sphinx-doc/sphinx/issues/701 for details on this issue. - -Decision --------- - -Copy monitoring/docs files to the main docs directory, and use an include statement as a quick temporary solution to publishing docs with the following benefits: - -* Original docs stay close to the code. -* Any old github references to the old locations would still be accurate. -* We may be able to implement a more automated solution in the future that doesn't break the new Readthedoc locations. - -Consequences ------------- - -New docs need to be copied to be included in the index.rst, which is only one minor additional step. - -Rejected Alternatives ---------------------- - -The following alternatives were temporarily rejected for the sake of expediency. The decision could be updated in more time were invested on this issue more globally. - -#. Move the monitoring/docs folder under the main docs folder. This would break existing references to github docs. Someone could choose to make this change in the future. -#. Create sphinx plugin to automatically copy docs. This is a potentially good idea, but I am not investing in it at this time. diff --git a/docs/monitoring/decisions/0001-monitoring-by-code-owner.rst b/docs/monitoring/decisions/0001-monitoring-by-code-owner.rst deleted file mode 100644 index fabbdac7..00000000 --- a/docs/monitoring/decisions/0001-monitoring-by-code-owner.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/decisions/0001-monitoring-by-code-owner.rst diff --git a/docs/monitoring/decisions/0002-custom-monitoring-language.rst b/docs/monitoring/decisions/0002-custom-monitoring-language.rst deleted file mode 100644 index 925082e8..00000000 --- a/docs/monitoring/decisions/0002-custom-monitoring-language.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/decisions/0002-custom-monitoring-language.rst diff --git a/docs/monitoring/decisions/0003-code-owner-for-celery-tasks.rst b/docs/monitoring/decisions/0003-code-owner-for-celery-tasks.rst deleted file mode 100644 index 89db44ee..00000000 --- a/docs/monitoring/decisions/0003-code-owner-for-celery-tasks.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst diff --git a/docs/monitoring/decisions/0004-code-owner-theme-and-squad.rst b/docs/monitoring/decisions/0004-code-owner-theme-and-squad.rst deleted file mode 100644 index 1f8881f0..00000000 --- a/docs/monitoring/decisions/0004-code-owner-theme-and-squad.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/decisions/0004-code-owner-theme-and-squad.rst diff --git a/docs/monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida.rst b/docs/monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida.rst deleted file mode 100644 index f451b649..00000000 --- a/docs/monitoring/how_tos/add_code_owner_custom_attribute_to_an_ida.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/how_tos/add_code_owner_custom_attribute_to_an_ida.rst diff --git a/docs/monitoring/how_tos/search_new_relic_nrql.rst b/docs/monitoring/how_tos/search_new_relic_nrql.rst deleted file mode 100644 index 0b304187..00000000 --- a/docs/monitoring/how_tos/search_new_relic_nrql.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/how_tos/search_new_relic_nrql.rst diff --git a/docs/monitoring/how_tos/update_monitoring_for_squad_or_theme_changes.rst b/docs/monitoring/how_tos/update_monitoring_for_squad_or_theme_changes.rst deleted file mode 100644 index 38976531..00000000 --- a/docs/monitoring/how_tos/update_monitoring_for_squad_or_theme_changes.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/how_tos/update_monitoring_for_squad_or_theme_changes.rst diff --git a/docs/monitoring/how_tos/using_custom_attributes.rst b/docs/monitoring/how_tos/using_custom_attributes.rst deleted file mode 100644 index 3f1d2fd0..00000000 --- a/docs/monitoring/how_tos/using_custom_attributes.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../edx_django_utils/monitoring/docs/how_tos/using_custom_attributes.rst diff --git a/edx_django_utils/plugins/README.rst b/edx_django_utils/plugins/README.rst index 3ba593ca..5655337e 100644 --- a/edx_django_utils/plugins/README.rst +++ b/edx_django_utils/plugins/README.rst @@ -27,12 +27,12 @@ into a containing Django project. Enable Plugins in an IDA ------------------------ -See :doc:`instructions to enable plugins for an IDA `. +See :doc:`instructions to enable plugins for an IDA `. Creating a Plugin App --------------------- -See :doc:`how to create a plugin app `. +See :doc:`how to create a plugin app `. .. note:: Do not use this plugin framework for required apps. Instead, explicitly add your required app to the ``INSTALLED_APPS`` of the IDA.