Minor documentation overhaul (#144)

* Updated Sphinx. Migrated from recommonmark to myst-parser. Removed a few hack from conf.py. * Removed most mentions of AFF4 from the docs. * Removed Python 2-specific sections. * Fixed the docs to suggest to change mysql server config file to set max_allowed_packet to 40M (see google/grr#947). * Largely rewritten "Install from source": now telling how to build Fleetspeak wheels and specifically mentioning the case when we install from source to build a client template (see google/grr#948). * Made minor changes to "Build custom client templates". You can see the version of the docs from this PR at https://grr-doc.readthedocs.io/en/sphinx-update/index.html.
google · Nov 24, 2021 · 9c2bebf · 9c2bebf
1 parent 4cc2792
commit 9c2bebf
Show file tree

Hide file tree

Showing 26 changed files with 331 additions and 321 deletions.
diff --git a/conf.py b/conf.py
@@ -21,20 +21,11 @@
 # sys.path.insert(0, os.path.abspath('.'))
 
 
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.napoleon',
-    'sphinx.ext.mathjax',
-    'recommonmark',
+    'myst_parser',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -75,95 +66,25 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
-
-# -- Options for HTML output ----------------------------------------------
-
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
 html_theme = 'sphinx_rtd_theme'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = []
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# This is required for the alabaster theme
-# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
-# html_sidebars = {
-#     '**': [
-#         'relations.html',  # needs 'show_related': True theme option to display
-#         'searchbox.html',
-#     ]
-# }
-
-
-# -- Options for HTMLHelp output ------------------------------------------
+html_theme_options = {
+  # Toc options
+  'collapse_navigation': True,
+  'sticky_navigation': True,
+  'navigation_depth': 4,
+  'includehidden': True,
+  'titles_only': True,
+}
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'GRRdoc'
 
 
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htmaster_dobp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, 'GRR.tex', u'GRR Documentation',
-     u'GRR team', 'manual'),
-]
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'grr', u'GRR Documentation',
-     [author], 1)
-]
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (master_doc, 'GRR', u'GRR Documentation',
-     author, 'GRR', 'One line description of project.',
-     'Miscellaneous'),
-]
-
-# Configure sphinx to convert markdown links (recommonmark is broken at the
-# moment).
+# Configure sphinx to replace predefined version variables.
 from docutils import nodes, transforms
 
 class ProcessLink(transforms.Transform):
@@ -172,16 +93,11 @@ class ProcessLink(transforms.Transform):
 
     text_replacements = {
         "__GRR_VERSION__": "3.4.5.1",
-        "__GRR_DEB_VERSION__": "3.4.5-1"
+        "__GRR_DEB_VERSION__": "3.4.5-1",
+        "__FLEETSPEAK_PIP_VERSION__": "0.1.11"
     }
 
     def find_replace(self, node):
-        if isinstance(node, nodes.reference) and "refuri" in node:
-            r = node["refuri"]
-            if r.endswith(".md"):
-                r = r[:-3] + ".html"
-                node["refuri"] = r
-
         if isinstance(node, nodes.Text):
             for k, v in self.text_replacements.items():
                 if k in node.astext():
@@ -204,12 +120,5 @@ def apply(self):
         self.current_level = 0
         self.traverse(self.document)
 
-from recommonmark.transform import AutoStructify
-
 def setup(app):
-    app.add_config_value('recommonmark_config', {
-        'enable_auto_toc_tree': True,
-        'auto_toc_tree_section': 'Table of contents',
-    }, True)
-    app.add_transform(AutoStructify)
     app.add_transform(ProcessLink)
diff --git a/deploying-grr-clients/index.md b/deploying-grr-clients/index.md
@@ -1,10 +1,16 @@
 # Deploying GRR clients
 
-* [Overview](overview.md)
-* [How to check if a deployed client talks back to GRR server](how-to-check-if-client-talks-back.md)
-* [On Windows](on-windows.md)
-* [On Mac OS X](on-mac-os-x.md)
-* [On Linux](on-linux.md)
-* [Life of a GRR client (what happens after deployment)](life-of-grr-client.md)
-* [Client Protection](protection.md)
-* [Troubleshooting ("I don't see my clients")](troubleshooting.md)
+```{toctree}
+---
+maxdepth: 1
+---
+Overview <overview.md>
+How to check if a deployed client talks back to GRR server <how-to-check-if-client-talks-back.md>
+On Windows <on-windows.md>
+On macOS <on-mac-os-x.md>
+On Linux <on-linux.md>
+Life of a GRR client (what happens after deployment) <life-of-grr-client.md>
+Client Protection <protection.md>
+Troubleshooting ("I don't see my clients") <troubleshooting.md>
+```
+
diff --git a/developing-grr/implementation-details/index.md b/developing-grr/implementation-details/index.md
@@ -1,6 +1,11 @@
 # Implementation details
 
-* [GRR messages](grr-messages.md)
-* [Client Path Specification](client-path-spec.md)
-* [Foreman](foreman.md)
-* [Authorization and Auditing](authorizating-and-auditing.md)
+```{toctree}
+---
+maxdepth: 1
+---
+GRR messages <grr-messages.md>
+Client Path Specification <client-path-spec.md>
+Foreman <foreman.md>
+Authorization and Auditing <authorizating-and-auditing.md>
+```
diff --git a/developing-grr/index.md b/developing-grr/index.md
@@ -1,7 +1,12 @@
 # Developing GRR
 
-* [Setting up development environment](setting-up-dev-env.md)
-* [Running the tests](running-tests.md)
-* [Contributing code](contributing.md)
-* [Contributing documentation](contributing-docs.md)
-* [Implementation details](implementation-details/index.md)
+```{toctree}
+---
+maxdepth: 1
+---
+Setting up development environment <setting-up-dev-env.md>
+Running the tests <running-tests.md>
+Contributing code <contributing.md>
+Contributing documentation <contributing-docs.md>
+Implementation details <implementation-details/index.md>
+```
diff --git a/fleetspeak/index.md b/fleetspeak/index.md
@@ -1,5 +1,10 @@
 # GRR and Fleetspeak setup
 
-* [Installing from a release DEB (recommended)](from-release-deb.md)
-* [Installing from source](from-source.md)
-* [Scaling a fleetspeak setup](scaling.md)
+```{toctree}
+---
+maxdepth: 1
+---
+Installing from a release DEB (recommended) <from-release-deb.md>
+Installing from source <from-source.md>
+Scaling a fleetspeak setup <scaling.md>
+```
diff --git a/index.md b/index.md
@@ -18,14 +18,20 @@ GRR is open source (Apache License 2.0) and is developed on GitHub: [github.com/
 
 ## Table of contents
 
-* [What is GRR?](what-is-grr.md)
-* [Quickstart](quickstart.md)
-* [Installing GRR server](installing-grr-server/index.md)
-* [Deploying GRR clients](deploying-grr-clients/index.md)
-* [Investigating with GRR](investigating-with-grr/index.md)
-* [Maintaining and tuning GRR deployment](maintaining-and-tuning/index.md)
-* [GRR and Fleetspeak setup](fleetspeak/index.md)
-* [Developing GRR](developing-grr/index.md)
-* [Release Notes](release-notes.md)
-* [F.A.Q](faq.md)
-* [Publications](publications.md)
+```{toctree}
+---
+maxdepth: 2
+---
+What is GRR? <what-is-grr.md>
+Quickstart <quickstart.md>
+Installing GRR server <installing-grr-server/index.md>
+Deploying GRR clients <deploying-grr-clients/index.md>
+Investigating with GRR <investigating-with-grr/index.md>
+Maintaining and tuning GRR deployment <maintaining-and-tuning/index.md>
+GRR and Fleetspeak setup <fleetspeak/index.md>
+Developing GRR <developing-grr/index.md>
+Release Notes <release-notes.md>
+F.A.Q <faq.md>
+Publications <publications.md>
+```
+
diff --git a/installing-grr-server/from-head-deb.md b/installing-grr-server/from-head-deb.md
@@ -1,9 +1,3 @@
-***Note on the AFF4 datastore deprecation***
-
-*Starting from the version ***3.3.0.0*** GRR uses a new datastore format by default - ***REL_DB***. REL_DB is backwards-incompatible with the now-deprecated AFF4 datastore format (even though they both use MySQL as a backend).*
-
-*Use of AFF4-based deployments is now discouraged. REL_DB is expected to be much more stable and performant. Please see [these docs](../maintaining-and-tuning/grr-datastore.md) if you're upgrading an older GRR version and would like to try out the new datastore.*
-
 # Installing from a HEAD DEB
 
 1. If the Google Cloud SDK is not already installed, download a tarball for

diff --git a/installing-grr-server/from-release-deb.md b/installing-grr-server/from-release-deb.md
@@ -1,14 +1,8 @@
-***Note on the AFF4 datastore deprecation***
-
-*Starting from the version ***3.3.0.0*** GRR uses a new datastore format by default - ***REL_DB***. REL_DB is backwards-incompatible with the now-deprecated AFF4 datastore format (even though they both use MySQL as a backend).*
-
-*Use of AFF4-based deployments is now discouraged. REL_DB is expected to be much more stable and performant. Please see [these docs](../maintaining-and-tuning/grr-datastore.md) if you're upgrading an older GRR version and would like to try out the new datastore.*
-
 # Installing from a release server deb (recommended)
 
 This is the recommended way of installing the GRR server components. GRR server
-debs are built for Ubuntu 18.04 Bionic. They may install on Debian or other Ubuntu
-versions, but compatibility is not guaranteed.
+debs are built for Ubuntu 18.04 Bionic (x86-64). They may install on Debian or other
+Ubuntu versions, but compatibility is not guaranteed.
 
 1. MySQL is GRR's default database backend, and should be up and running
 before installing GRR. The database framework can be run alongside GRR on the
@@ -21,7 +15,22 @@ community edition of MySQL from Ubuntu repositories:
 
     If you have never installed MySQL on the machine before, you will be
     prompted for a password for the 'root' database user. After installation
-    completes, you will typically want to create a new database
+    completes, you need to change the `max_allowed_packet` setting, otherwise
+    GRR will have issues writing large chunks of data to the database. Put
+    the following into `/etc/mysql/my.cnf`:
+
+    ```
+    [mysqld]
+    max_allowed_packet=40M
+    ```
+
+    Then restart the MySQL server:
+
+    ```
+    service mysql restart
+    ```
+
+    You will typically want to create a new database
     user for GRR and give that user access an empty database that
     the GRR server installation will use:
 
@@ -30,8 +39,6 @@ community edition of MySQL from Ubuntu repositories:
     ```
 
     ```bash
-    mysql> SET GLOBAL max_allowed_packet=41943040;
-
     mysql> CREATE USER 'grr'@'localhost' IDENTIFIED BY 'password';
 
     mysql> CREATE DATABASE grr;

diff --git a/installing-grr-server/from-released-pip.md b/installing-grr-server/from-released-pip.md
@@ -1,9 +1,3 @@
-***Note on the AFF4 datastore deprecation***
-
-*Starting from the version ***3.3.0.0*** GRR uses a new datastore format by default - ***REL_DB***. REL_DB is backwards-incompatible with the now-deprecated AFF4 datastore format (even though they both use MySQL as a backend).*
-
-*Use of AFF4-based deployments is now discouraged. REL_DB is expected to be much more stable and performant. Please see [these docs](../maintaining-and-tuning/grr-datastore.md) if you're upgrading an older GRR version and would like to try out the new datastore.*
-
 # Installing from released PIP packages
 
 If the templates included in release server debs are not