From 7c04731d64bc6a965bfbf6a3d2ba13aa40fc03ab Mon Sep 17 00:00:00 2001 From: Steven Terrana Date: Thu, 29 Sep 2022 11:48:02 -0400 Subject: [PATCH 1/4] add dependency on GitPython --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 54d6b1e..8d669fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ ] dependencies = [ "mkdocs>=1.0.4", + "GitPython>=3.1.27" ] [project.entry-points."mkdocs.plugins"] From 3d02e37aa070f3dd9e24060fbb4a1820a877f08b Mon Sep 17 00:00:00 2001 From: Steven Terrana Date: Thu, 29 Sep 2022 11:50:37 -0400 Subject: [PATCH 2/4] resolve cleanup bugs --- yamp/plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/yamp/plugin.py b/yamp/plugin.py index 84ed8a2..a9f1962 100644 --- a/yamp/plugin.py +++ b/yamp/plugin.py @@ -67,12 +67,14 @@ def on_config(self, config): # aggregates documentation def on_pre_build(self, config): + # the actual directory where we'll add the repositories + self._temp_dir = os.path.join(config.docs_dir, self.config.temp_dir) + # wipe temp_dir on first build if self.first_build and self.config.start_fresh: self.cleanup() # create repos directory if it doesn't exist - self._temp_dir = os.path.join(config.docs_dir, self.config.temp_dir) if not os.path.exists(self._temp_dir): os.makedirs(self._temp_dir) for repo in self.config.repos: @@ -89,8 +91,8 @@ def on_pre_page(self, page, config, files): filtered[0].set_edit_url(page, self.config.temp_dir) def cleanup(self): - if self.config.cleanup: - shutil.rmtree(self.basedir) + if self.config.cleanup and os.path.exists(self._temp_dir): + shutil.rmtree(self._temp_dir) def on_shutdown(self): self.cleanup() \ No newline at end of file From 3352766e162637a4d27dba0ad7f1080eea98d03a Mon Sep 17 00:00:00 2001 From: Steven Terrana Date: Thu, 29 Sep 2022 11:57:38 -0400 Subject: [PATCH 3/4] add newline to EOF --- yamp/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamp/plugin.py b/yamp/plugin.py index a9f1962..bda021c 100644 --- a/yamp/plugin.py +++ b/yamp/plugin.py @@ -95,4 +95,4 @@ def cleanup(self): shutil.rmtree(self._temp_dir) def on_shutdown(self): - self.cleanup() \ No newline at end of file + self.cleanup() From 47ac8db9cd477df7d6fcb0599987f5e5e4750819 Mon Sep 17 00:00:00 2001 From: Steven Terrana Date: Thu, 29 Sep 2022 12:29:02 -0400 Subject: [PATCH 4/4] fix pylint issues --- .gitignore | 3 +- yamp/plugin.py | 166 +++++++++++++++++++----------------- yamp/repo_item.py | 208 +++++++++++++++++++++++++--------------------- 3 files changed, 204 insertions(+), 173 deletions(-) diff --git a/.gitignore b/.gitignore index 4d457b7..b64ef6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ docs/repos site/ dist/ -.pypi.env \ No newline at end of file +.pypi.env +.venv \ No newline at end of file diff --git a/yamp/plugin.py b/yamp/plugin.py index bda021c..0a0cf80 100644 --- a/yamp/plugin.py +++ b/yamp/plugin.py @@ -1,13 +1,15 @@ -# Copyright 2022 Booz Allen Hamilton -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +""" +Copyright 2022 Booz Allen Hamilton +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" import os import shutil @@ -24,75 +26,83 @@ log.addFilter(warning_filter) class YAMPConfig(Config): - # determines whether the generated `temp_dir` should be - # deleted after the mkdocs invocation - cleanup = c.Type(bool, default=True) + """defines the plugin configuration""" + # determines whether the generated `temp_dir` should be + # deleted after the mkdocs invocation + cleanup = c.Type(bool, default=True) - # determines whether the `temp_dir` should be deleted - # if it exists as the start of the mkdocs invocation - start_fresh = c.Type(bool, default=True) - - # the local path within the docs directory where - # repositories should be cloned - temp_dir = c.Type(str, default="repos") + # determines whether the `temp_dir` should be deleted + # if it exists as the start of the mkdocs invocation + start_fresh = c.Type(bool, default=True) - # the list of repositories to clone - repos = c.ListOfItems(c.SubConfig(RepoItem), default = []) + # the local path within the docs directory where + # repositories should be cloned + temp_dir = c.Type(str, default="repos") -class YAMP(BasePlugin[YAMPConfig]): + # the list of repositories to clone + repos = c.ListOfItems(c.SubConfig(RepoItem), default = []) - first_build = True - # the actual Directory defined by self.config.temp_dir - _temp_dir = None - - def log(self, message): - log.info(f'[repo aggregator] {message}') - - def __init__(self): - self.enabled = True - - # registers the plugin to persist a common - # instance across builds during mkdocs serve - def on_startup(self, command, dirty): - pass - - # validates the repo configurations - def on_config(self, config): - for repo in self.config.repos: - try: - repo.do_validation() - except: - log.warning(f'misconfigured repo: {repo}') - raise - - # aggregates documentation - def on_pre_build(self, config): - # the actual directory where we'll add the repositories - self._temp_dir = os.path.join(config.docs_dir, self.config.temp_dir) - - # wipe temp_dir on first build - if self.first_build and self.config.start_fresh: - self.cleanup() - - # create repos directory if it doesn't exist - if not os.path.exists(self._temp_dir): - os.makedirs(self._temp_dir) - for repo in self.config.repos: - repo.fetch(self._temp_dir, self.first_build) - - self.first_build = False - - # change the page's edit URL if the page came from - # one of the define repositories - def on_pre_page(self, page, config, files): - path = page.file.src_path - filtered = [ repo for repo in self.config.repos if path.startswith(f'{self.config.temp_dir}/{repo.repo_name}') ] - if len(filtered) > 0: - filtered[0].set_edit_url(page, self.config.temp_dir) - - def cleanup(self): - if self.config.cleanup and os.path.exists(self._temp_dir): - shutil.rmtree(self._temp_dir) - - def on_shutdown(self): - self.cleanup() +class YAMP(BasePlugin[YAMPConfig]): + """Aggregates repositories defined by users in the mkdocs.yaml""" + + first_build = True + # the actual Directory defined by self.config.temp_dir + _temp_dir = None + + def __init__(self): + self.enabled = True + + # registers the plugin to persist a common + # instance across builds during mkdocs serve + def on_startup(self, _command, _dirty): + """ + registers this plugin instance to persist across builds during mkdocs serve + """ + + # validates the repo configurations + def on_config(self, _config): + """validates the repo configurations""" + for repo in self.config.repos: + try: + repo.do_validation() + except: + log.warning('misconfigured repo: %s', repo) + raise + + def on_pre_build(self, config): + """aggregates documentation""" + # the actual directory where we'll add the repositories + self._temp_dir = os.path.join(config.docs_dir, self.config.temp_dir) + + # wipe temp_dir on first build + if self.first_build and self.config.start_fresh: + self.cleanup() + + # create repos directory if it doesn't exist + if not os.path.exists(self._temp_dir): + os.makedirs(self._temp_dir) + for repo in self.config.repos: + repo.fetch(self._temp_dir, self.first_build) + + self.first_build = False + + def on_pre_page(self, page, _config, _files): + """ + Change the page's edit URL if the page came from one of the defined repositories + """ + path = page.file.src_path + filtered = [ + repo for repo in self.config.repos + if path.startswith(f'{self.config.temp_dir}/{repo.repo_name}') + ] + if len(filtered) > 0: + filtered[0].set_edit_url(page, self.config.temp_dir) + + def cleanup(self): + """deletes the temporary directory where repos are aggregated""" + if self.config.cleanup and os.path.exists(self._temp_dir): + shutil.rmtree(self._temp_dir) + + def on_shutdown(self): + """cleanup at the end of the mkdocs invocation""" + self.cleanup() diff --git a/yamp/repo_item.py b/yamp/repo_item.py index f82b8e6..e044dc6 100644 --- a/yamp/repo_item.py +++ b/yamp/repo_item.py @@ -1,13 +1,15 @@ -# Copyright 2022 Booz Allen Hamilton -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +""" +Copyright 2022 Booz Allen Hamilton +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" import os import logging @@ -21,88 +23,106 @@ log.addFilter(warning_filter) class RepoItem(Config): - ### Configuration provided by mkdocs.yaml - # the repository URL to clone - url = c.Optional(c.Type(str, default = None)) - # the branch of the repository to clone - branch = c.Type(str, default = "main") - # a list of globs specifying paths within - # the repository to clone - include = c.ListOfItems(c.Type(str), default = []) - - # a relative path from the mkdocs.yaml file - # to a directory to include in the generated - # 'temp_dir' directory via symlink - path = c.Optional(c.Type(str)) - - ### Configuration determined by self - repo_name = None - - # validate the user-provided configuration - def do_validation(self): - # you must define either a URL or a path, but not both - if not (self.url or self.path): - raise PluginError(f'repo does not define a url or a path') - - if self.url and self.path: - raise PluginError(f'repo cannot define both a url and a path') - - def fetch(self, temp_dir, first_build): - if self.url: - self.clone_git_repo(temp_dir, first_build) - else: - self.create_symlink(temp_dir) - - def clone_git_repo(self, temp_dir, first_build): - self.repo_name = self.url.split("/")[-1].replace('.git','') - # path to clone the repository to - r_path = os.path.join(temp_dir, self.repo_name) - if os.path.exists(r_path): - # # do a git pull - self.log.info(f'git pull {self.url}') - # r = Repo(r_path) - # r.remotes.origin.pull() - else: - # do a git clone - if self.include: - # equivalent to: - # git clone --no-checkout $repo - # cd $repo - # git checkout origin/main -- some_file.md some_directory/ - r = Repo.clone_from(self.url, r_path, no_checkout = True) - if not self.branchExists(r, self.branch): - raise PluginError(f'repository {self.url} does not have branch {self.branch}') + """represents a repository defined by the user + + defines both the plugin configuration schema and + performs actions like cloning repositories or + creating symlinks + """ + ### Configuration provided by mkdocs.yaml + # the repository URL to clone + url = c.Optional(c.Type(str, default = None)) + + # the branch of the repository to clone + branch = c.Type(str, default = "main") + + # a list of globs specifying paths within + # the repository to clone + include = c.ListOfItems(c.Type(str), default = []) + + # a relative path from the mkdocs.yaml file + # to a directory to include in the generated + # 'temp_dir' directory via symlink + path = c.Optional(c.Type(str)) + + ### Configuration determined by self + # the name of the sub directory created within the defined temp_dir + repo_name = None + + # validate the user-provided configuration + def do_validation(self): + """validates the user configuration""" + # you must define either a URL or a path, but not both + if not (self.url or self.path): + raise PluginError('repo does not define a url or a path') + + if self.url and self.path: + raise PluginError('repo cannot define both a url and a path') + + def fetch(self, temp_dir, first_build): + """adds the repositories contents to temp_dir""" + if self.url: + self.clone_git_repo(temp_dir, first_build) else: - git = r.git() - git.checkout(f'origin/{self.branch}', "--", *self.include) - else: - r = Repo.clone_from(self.url, r_path) - if not self.branchExists(r, self.branch): - raise PluginError(f'repository {self.url} does not have branch {self.branch}') - r.git.checkout(f'origin/{self.branch}') - - def branchExists(self, repo, branch): - return f'origin/{branch}' in [ ref.name for ref in repo.references ] - - def set_edit_url(self, page, temp_dir): - if self.url: - prefix = f'{temp_dir}/{self.repo_name}/' - edit_url = ''.join([ - self.url.replace(".git", ""), - "/edit/", - f'{self.branch}/', - page.file.src_path[len(prefix):] - ]) - page.edit_url = edit_url - else: - page.edit_url = None - - def create_symlink(self, temp_dir): - src = os.path.abspath(self.path) - if not os.path.exists(src): - raise PluginError(f'path {src} does not exist') - - self.repo_name = os.path.basename(src) - dst = os.path.abspath(os.path.join(temp_dir, self.repo_name)) - if not os.path.exists(dst): - os.symlink(src, dst, True) \ No newline at end of file + self.create_symlink(temp_dir) + + def clone_git_repo(self, temp_dir, _first_build): + """clones a remote git repository""" + self.repo_name = self.url.split("/")[-1].replace('.git','') + + # path to clone the repository to + r_path = os.path.join(temp_dir, self.repo_name) + + if os.path.exists(r_path): + # # do a git pull + self.log.info(f'git pull {self.url}') + # r = Repo(r_path) + # r.remotes.origin.pull() + else: + # do a git clone + if self.include: + # equivalent to: + # git clone --no-checkout $repo + # cd $repo + # git checkout origin/main -- some_file.md some_directory/ + cloned_repo = Repo.clone_from(self.url, r_path, no_checkout = True) + + if not self.branch_exists(cloned_repo, self.branch): + raise PluginError(f'repository {self.url} does not have branch {self.branch}') + + git = cloned_repo.git() + git.checkout(f'origin/{self.branch}', "--", *self.include) + else: + cloned_repo = Repo.clone_from(self.url, r_path) + if not self.branch_exists(cloned_repo, self.branch): + raise PluginError(f'repository {self.url} does not have branch {self.branch}') + cloned_repo.git.checkout(f'origin/{self.branch}') + + def branch_exists(self, repo, branch): + """determines if the user-provided branch exists in the remote repository""" + return f'origin/{branch}' in [ ref.name for ref in repo.references ] + + def set_edit_url(self, page, temp_dir): + """changes the edit URL if the page comes from a remote repository""" + if self.url: + prefix = f'{temp_dir}/{self.repo_name}/' + edit_url = ''.join([ + self.url.replace(".git", ""), + "/edit/", + f'{self.branch}/', + page.file.src_path[len(prefix):] + ]) + page.edit_url = edit_url + else: + page.edit_url = None + + def create_symlink(self, temp_dir): + """creates a symlink within temp_dir to the user provided path""" + src = os.path.abspath(self.path) + if not os.path.exists(src): + raise PluginError(f'path {src} does not exist') + + self.repo_name = os.path.basename(src) + dst = os.path.abspath(os.path.join(temp_dir, self.repo_name)) + if not os.path.exists(dst): + os.symlink(src, dst, True)