diff --git a/docs/async_data_client.rst b/docs/async_data_client/async_data_client.rst similarity index 52% rename from docs/async_data_client.rst rename to docs/async_data_client/async_data_client.rst index 7d2901de4..c5cc70740 100644 --- a/docs/async_data_client.rst +++ b/docs/async_data_client/async_data_client.rst @@ -1,6 +1,6 @@ Bigtable Data Client Async ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: google.cloud.bigtable.data._async.client +.. autoclass:: google.cloud.bigtable.data._async.client.BigtableDataClientAsync :members: :show-inheritance: diff --git a/docs/async_data_exceptions.rst b/docs/async_data_client/async_data_exceptions.rst similarity index 100% rename from docs/async_data_exceptions.rst rename to docs/async_data_client/async_data_exceptions.rst diff --git a/docs/async_data_mutations.rst b/docs/async_data_client/async_data_mutations.rst similarity index 100% rename from docs/async_data_mutations.rst rename to docs/async_data_client/async_data_mutations.rst diff --git a/docs/async_data_mutations_batcher.rst b/docs/async_data_client/async_data_mutations_batcher.rst similarity index 100% rename from docs/async_data_mutations_batcher.rst rename to docs/async_data_client/async_data_mutations_batcher.rst diff --git a/docs/async_data_read_modify_write_rules.rst b/docs/async_data_client/async_data_read_modify_write_rules.rst similarity index 100% rename from docs/async_data_read_modify_write_rules.rst rename to docs/async_data_client/async_data_read_modify_write_rules.rst diff --git a/docs/async_data_read_rows_query.rst b/docs/async_data_client/async_data_read_rows_query.rst similarity index 100% rename from docs/async_data_read_rows_query.rst rename to docs/async_data_client/async_data_read_rows_query.rst diff --git a/docs/async_data_row.rst b/docs/async_data_client/async_data_row.rst similarity index 100% rename from docs/async_data_row.rst rename to docs/async_data_client/async_data_row.rst diff --git a/docs/async_data_row_filters.rst b/docs/async_data_client/async_data_row_filters.rst similarity index 100% rename from docs/async_data_row_filters.rst rename to docs/async_data_client/async_data_row_filters.rst diff --git a/docs/async_data_client/async_data_table.rst b/docs/async_data_client/async_data_table.rst new file mode 100644 index 000000000..a977beb6a --- /dev/null +++ b/docs/async_data_client/async_data_table.rst @@ -0,0 +1,6 @@ +Table Async +~~~~~~~~~~~ + +.. autoclass:: google.cloud.bigtable.data._async.client.TableAsync + :members: + :show-inheritance: diff --git a/docs/async_data_usage.rst b/docs/async_data_client/async_data_usage.rst similarity index 80% rename from docs/async_data_usage.rst rename to docs/async_data_client/async_data_usage.rst index c436c5988..8843b506b 100644 --- a/docs/async_data_usage.rst +++ b/docs/async_data_client/async_data_usage.rst @@ -1,10 +1,11 @@ -Using the Async Data Client -=========================== +Async Data Client +================= .. toctree:: :maxdepth: 2 async_data_client + async_data_table async_data_mutations_batcher async_data_read_rows_query async_data_row diff --git a/docs/app-profile.rst b/docs/classic_client/app-profile.rst similarity index 100% rename from docs/app-profile.rst rename to docs/classic_client/app-profile.rst diff --git a/docs/backup.rst b/docs/classic_client/backup.rst similarity index 100% rename from docs/backup.rst rename to docs/classic_client/backup.rst diff --git a/docs/batcher.rst b/docs/classic_client/batcher.rst similarity index 100% rename from docs/batcher.rst rename to docs/classic_client/batcher.rst diff --git a/docs/client-intro.rst b/docs/classic_client/client-intro.rst similarity index 100% rename from docs/client-intro.rst rename to docs/classic_client/client-intro.rst diff --git a/docs/client.rst b/docs/classic_client/client.rst similarity index 100% rename from docs/client.rst rename to docs/classic_client/client.rst diff --git a/docs/cluster.rst b/docs/classic_client/cluster.rst similarity index 100% rename from docs/cluster.rst rename to docs/classic_client/cluster.rst diff --git a/docs/column-family.rst b/docs/classic_client/column-family.rst similarity index 100% rename from docs/column-family.rst rename to docs/classic_client/column-family.rst diff --git a/docs/data-api.rst b/docs/classic_client/data-api.rst similarity index 100% rename from docs/data-api.rst rename to docs/classic_client/data-api.rst diff --git a/docs/encryption-info.rst b/docs/classic_client/encryption-info.rst similarity index 100% rename from docs/encryption-info.rst rename to docs/classic_client/encryption-info.rst diff --git a/docs/instance-api.rst b/docs/classic_client/instance-api.rst similarity index 100% rename from docs/instance-api.rst rename to docs/classic_client/instance-api.rst diff --git a/docs/instance.rst b/docs/classic_client/instance.rst similarity index 100% rename from docs/instance.rst rename to docs/classic_client/instance.rst diff --git a/docs/row-data.rst b/docs/classic_client/row-data.rst similarity index 100% rename from docs/row-data.rst rename to docs/classic_client/row-data.rst diff --git a/docs/row-filters.rst b/docs/classic_client/row-filters.rst similarity index 100% rename from docs/row-filters.rst rename to docs/classic_client/row-filters.rst diff --git a/docs/row-set.rst b/docs/classic_client/row-set.rst similarity index 100% rename from docs/row-set.rst rename to docs/classic_client/row-set.rst diff --git a/docs/row.rst b/docs/classic_client/row.rst similarity index 100% rename from docs/row.rst rename to docs/classic_client/row.rst diff --git a/docs/snippets.py b/docs/classic_client/snippets.py similarity index 100% rename from docs/snippets.py rename to docs/classic_client/snippets.py diff --git a/docs/snippets_table.py b/docs/classic_client/snippets_table.py similarity index 100% rename from docs/snippets_table.py rename to docs/classic_client/snippets_table.py diff --git a/docs/table-api.rst b/docs/classic_client/table-api.rst similarity index 100% rename from docs/table-api.rst rename to docs/classic_client/table-api.rst diff --git a/docs/table.rst b/docs/classic_client/table.rst similarity index 100% rename from docs/table.rst rename to docs/classic_client/table.rst diff --git a/docs/usage.rst b/docs/classic_client/usage.rst similarity index 91% rename from docs/usage.rst rename to docs/classic_client/usage.rst index de0abac9c..7a47f4d4a 100644 --- a/docs/usage.rst +++ b/docs/classic_client/usage.rst @@ -1,10 +1,15 @@ -Using the Sync Client -===================== +Classic Client +============== .. toctree:: :maxdepth: 2 client-intro + + instance-api + table-api + data-api + client cluster instance diff --git a/docs/index.rst b/docs/index.rst index 0f04542cc..4204e981d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,29 +2,19 @@ .. include:: multiprocessing.rst -Using the API +Client Types ------------- .. toctree:: :maxdepth: 2 - usage - async_data_usage - - -API Reference -------------- -.. toctree:: - :maxdepth: 2 - - instance-api - table-api - data-api + classic_client/usage + async_data_client/async_data_usage Changelog --------- -For a list of all ``google-cloud-datastore`` releases: +For a list of all ``google-cloud-bigtable`` releases: .. toctree:: :maxdepth: 2 diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py new file mode 100644 index 000000000..6338128dd --- /dev/null +++ b/docs/scripts/patch_devsite_toc.py @@ -0,0 +1,201 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script will run after ``nox -s docfx`` is run. docfx is the api doc format used by +google cloud. It is described here: https://github.com/googleapis/docuploader?tab=readme-ov-file#requirements-for-docfx-yaml-tarballs. + +One of the file used by docfx is toc.yml which is used to generate the table of contents sidebar. +This script will patch file to create subfolders for each of the clients +""" + + +import yaml +import os +import shutil + +# set working directory to /docs +os.chdir(f"{os.path.dirname(os.path.abspath(__file__))}/{os.pardir}") + + +def add_sections(toc_file_path, section_list, output_file_path=None): + """ + Add new sections to the autogenerated docfx table of contents file + + Takes in a list of TocSection objects, which should point to a directory of rst files + within the main /docs directory, which represents a self-contained section of content + + :param toc_file_path: path to the autogenerated toc file + :param section_list: list of TocSection objects to add + :param output_file_path: path to save the updated toc file. If None, save to the input file + """ + # remove any sections that are already in the toc + remove_sections(toc_file_path, [section.title for section in section_list]) + # add new sections + current_toc = yaml.safe_load(open(toc_file_path, "r")) + for section in section_list: + print(f"Adding section {section.title}...") + current_toc[0]["items"].insert(-1, section.to_dict()) + section.copy_markdown() + # save file + if output_file_path is None: + output_file_path = toc_file_path + with open(output_file_path, "w") as f: + yaml.dump(current_toc, f) + + +def remove_sections(toc_file_path, section_list, output_file_path=None): + """ + Remove sections from the autogenerated docfx table of contents file + + Takes in a list of string section names to remove from the toc file + + :param toc_file_path: path to the autogenerated toc file + :param section_list: list of section names to remove + :param output_file_path: path to save the updated toc file. If None, save to the input file + """ + current_toc = yaml.safe_load(open(toc_file_path, "r")) + print(f"Removing sections {section_list}...") + new_items = [d for d in current_toc[0]["items"] if d["name"] not in section_list] + current_toc[0]["items"] = new_items + # save file + if output_file_path is None: + output_file_path = toc_file_path + with open(output_file_path, "w") as f: + yaml.dump(current_toc, f) + + +class TocSection: + def __init__(self, dir_name, index_file_name): + """ + :param dir_name: name of the directory containing the rst files + :param index_file_name: name of an index file within dir_name. This file + will not be included in the table of contents, but provides an ordered + list of the other files which should be included + """ + self.dir_name = dir_name + self.index_file_name = index_file_name + index_file_path = os.path.join(dir_name, index_file_name) + # find set of files referenced by the index file + with open(index_file_path, "r") as f: + self.title = f.readline().strip() + in_toc = False + self.items = [] + for line in f: + # ignore empty lines + if not line.strip(): + continue + if line.startswith(".. toctree::"): + in_toc = True + continue + # ignore directives + if ":" in line: + continue + if not in_toc: + continue + # bail when toc indented block is done + if not line.startswith(" ") and not line.startswith("\t"): + break + # extract entries + self.items.append(self.extract_toc_entry(line.strip())) + + def extract_toc_entry(self, file_name): + """ + Given the name of a file, extract the title and href for the toc entry, + and return as a dictionary + """ + # load the file to get the title + with open(f"{self.dir_name}/{file_name}.rst", "r") as f2: + file_title = f2.readline().strip() + return {"name": file_title, "href": f"{file_name}.md"} + + def to_dict(self): + """ + Convert the TocSection object to a dictionary that can be written to a yaml file + """ + return {"name": self.title, "items": self.items} + + def copy_markdown(self): + """ + Copy markdown files from _build/markdown/dir_name to _build/html/docfx_yaml + + This is necessary because the markdown files in sub-directories + are not copied over by the docfx build by default + """ + for file in os.listdir("_build/markdown/" + self.dir_name): + shutil.copy( + f"_build/markdown/{self.dir_name}/{file}", + f"_build/html/docfx_yaml", + ) + + +def validate_toc(toc_file_path, expected_section_list, added_sections): + current_toc = yaml.safe_load(open(toc_file_path, "r")) + # make sure the set of sections matches what we expect + found_sections = [d["name"] for d in current_toc[0]["items"]] + assert found_sections == expected_section_list + # make sure each customs ection is in the toc + for section in added_sections: + assert section.title in found_sections + # make sure each rst file in each custom section dir is listed in the toc + for section in added_sections: + items_in_toc = [ + d["items"] + for d in current_toc[0]["items"] + if d["name"] == section.title and ".rst" + ][0] + items_in_dir = [f for f in os.listdir(section.dir_name) if f.endswith(".rst")] + # subtract 1 for index + assert len(items_in_toc) == len(items_in_dir) - 1 + for file in items_in_dir: + if file != section.index_file_name: + base_name, _ = os.path.splitext(file) + assert any(d["href"] == f"{base_name}.md" for d in items_in_toc) + # make sure the markdown files are present in the docfx_yaml directory + for section in added_sections: + items_in_toc = [ + d["items"] + for d in current_toc[0]["items"] + if d["name"] == section.title and ".rst" + ][0] + md_files = [d["href"] for d in items_in_toc] + for file in md_files: + assert os.path.exists(f"_build/html/docfx_yaml/{file}") + print("Toc validation passed") + + +if __name__ == "__main__": + # Add secrtions for the async_data_client and classic_client directories + toc_path = "_build/html/docfx_yaml/toc.yml" + custom_sections = [ + TocSection( + dir_name="async_data_client", index_file_name="async_data_usage.rst" + ), + TocSection(dir_name="classic_client", index_file_name="usage.rst"), + ] + add_sections(toc_path, custom_sections) + # Remove the Bigtable section, since it has duplicated data + remove_sections(toc_path, ["Bigtable"]) + # run validation to make sure yaml is structured as we expect + validate_toc( + toc_file_path=toc_path, + expected_section_list=[ + "Overview", + "bigtable APIs", + "Changelog", + "Multiprocessing", + "Async Data Client", + "Classic Client", + ], + added_sections=custom_sections, + ) diff --git a/noxfile.py b/noxfile.py index f175c66da..3ea12c187 100644 --- a/noxfile.py +++ b/noxfile.py @@ -425,6 +425,9 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + # Customization: Add extra sections to the table of contents for the Classic vs Async clients + session.install("pyyaml") + session.run("python", "docs/scripts/patch_devsite_toc.py") @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) diff --git a/owlbot.py b/owlbot.py index cde9fce64..170bc08d4 100644 --- a/owlbot.py +++ b/owlbot.py @@ -213,6 +213,22 @@ def mypy(session): ) +# add customization to docfx +docfx_postprocess = """ + # Customization: Add extra sections to the table of contents for the Classic vs Async clients + session.install("pyyaml") + session.run("python", "docs/scripts/patch_devsite_toc.py") +""" + +place_before( + "noxfile.py", + "@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)\n" + "def prerelease_deps(session):", + docfx_postprocess, + escape="()" +) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint_setup_py(session): ''',