diff --git a/.github/workflows/wiki.yaml b/.github/workflows/wiki.yaml new file mode 100644 index 000000000..4c6357a71 --- /dev/null +++ b/.github/workflows/wiki.yaml @@ -0,0 +1,78 @@ +name: Wiki +on: + push: + branches: + - master + paths: + - 'wiki/**' + pull_request: + paths: + - 'wiki/**' +env: + CLONE_URL: "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.wiki.git" + +jobs: + build: + name: Build Wiki Artifact + runs-on: ubuntu-latest + env: + TEMP_WIKI_DIR: "wiki/.rez-gen-wiki-tmp" + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Set up Python 2.7 + uses: actions/setup-python@v1 + with: + python-version: 2.7 + + - name: Build Wiki + run: | + git config --global color.ui always + if [ "${{ github.event_name == 'pull_request' }}" == "true" ] + then + BRANCH="${{ github.head_ref }}" + else + BRANCH="${{ github.ref }}" + fi + + python wiki/update-wiki.py \ + --keep-temp \ + --no-push \ + --github-branch="${BRANCH##refs*/}" \ + --wiki-dir="${{ env.TEMP_WIKI_DIR }}" \ + --wiki-url="${{ env.CLONE_URL }}" + + - uses: actions/upload-artifact@v1.0.0 + with: + name: wiki-markdown + path: ${{ env.TEMP_WIKI_DIR }} + + + publish: + name: Publish to GitHub Wiki + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' + needs: build + + steps: + - uses: actions/download-artifact@master + with: + name: wiki-markdown + path: . + + - name: Setup git config + run: | + git config --global color.ui always + git config --global user.name "github.com/${{ github.actor }}" + git config --global user.email "${{ github.actor }}@${{ github.sha }}" + + - name: Git commit and push + run: | + git commit \ + -m "Generated from GitHub "${{ github.workflow }}" Workflow" \ + -m "See https://github.com/${{ github.repository }}/runs/${GITHUB_ACTION}" \ + || exit 0 + git remote set-url origin "${{ env.CLONE_URL }}" + git push origin master diff --git a/src/rez/cli/_main.py b/src/rez/cli/_main.py index 53e307b24..ea1031cc7 100644 --- a/src/rez/cli/_main.py +++ b/src/rez/cli/_main.py @@ -70,10 +70,12 @@ def __call__(self, parser, args, values, option_string=None): sys.exit(0) -def run(command=None): - - sys.dont_write_bytecode = True +def setup_parser(): + """Create and setup parser for given rez command line interface. + Returns: + LazyArgumentParser: Argument parser for rez command. + """ parser = LazyArgumentParser("rez") parser.add_argument("-i", "--info", action=InfoAction, @@ -99,6 +101,13 @@ def run(command=None): help='', # required so that it can be setup later setup_subparser=SetupRezSubParser(module_name)) + return parser + + +def run(command=None): + + sys.dont_write_bytecode = True + # construct args list. Note that commands like 'rez-env foo' and # 'rez env foo' are equivalent if command: @@ -115,6 +124,7 @@ def run(command=None): else: arg_mode = None + parser = setup_parser() if arg_mode == "grouped": # args split into groups by '--' arg_groups = [[]] diff --git a/wiki/pages/Command-Line-Tools.md b/wiki/pages/Command-Line-Tools.md deleted file mode 100644 index 5d0ba185f..000000000 --- a/wiki/pages/Command-Line-Tools.md +++ /dev/null @@ -1,43 +0,0 @@ -## rez - -## rez-bind - -## rez-build - -## rez-config - -## rez-context - -## rez-depends - -## rez-diff - -## rez-env - -## rez-gui - -## rez-help - -## rez-interpret - -## rez-memcache - -## rez-plugins - -## rez-python - -## rez-release - -## rez-search - -## rez-selftest - -## rez-status - -## rez-suite - -## rez-test - -## rez-view - -## rez-yaml2py diff --git a/wiki/pages/Home.md b/wiki/pages/Home.md new file mode 100644 index 000000000..f7407dc55 --- /dev/null +++ b/wiki/pages/Home.md @@ -0,0 +1,112 @@ +[[media/rez_banner_128.png]] + +- [What Is Rez?](#what-is-rez) +- [The Basics](#the-basics) +- [Examples](#examples) + + +## What Is Rez? + +Rez is a cross-platform package manager with a difference. Using Rez you can create +standalone environments configured for a given set of packages. However, unlike many +other package managers, packages are not installed into these standalone environments. +Instead, all package versions are installed into a central repository, and standalone +environments reference these existing packages. This means that configured environments +are lightweight, and very fast to create, often taking just a few seconds to configure +despite containing hundreds of packages. + +

+ + +
Typical package managers install packages into an environment +

+ +
+

+ + +
Rez installs packages once, and configures environments dynamically +

+ +
+Rez takes a list of package requests, and constructs the target environment, resolving +all the necessary package dependencies. Any type of software package is supported - +compiled, python, applications and libraries. + + +## The Basics + +Packages are stored in repositories on disk. Each package has a single concise +definition file (*package.py*) that defines its dependencies, its commands (how it +configures the environment containing it), and other metadata. For example, the +following is the package definition file for the popular *requests* python module: + + name = "requests" + + version = "2.8.1" + + authors = ["Kenneth Reitz"] + + requires = [ + "python-2.7+" + ] + + def commands(): + env.PYTHONPATH.append("{root}/python") + +This package requires python-2.7 or greater. When used, the 'python' subdirectory +within its install location is appended to the PYTHONPATH environment variable. + +When an environment is created with the rez API or *rez-env* tool, a dependency +resolution algorithm tracks package requirements and resolves to a list of needed +packages. The commands from these packages are concatenated and evaluated, resulting +in a configured environment. Rez is able to configure environments containing +hundreds of packages, often within a few seconds. Resolves can also be saved to file, +and when re-evaluated later will reconstruct the same environment once more. + + +## Examples + +This example places the user into a resolved shell containing the requested packages, +using the [rez-env](https://github.com/nerdvegas/rez/wiki/Command-Line-Tools#rez-env) tool: + + ]$ rez-env requests-2.2+ python-2.6 'pymongo-0+<2.7' + + You are now in a rez-configured environment. + + resolved by ajohns@nn188.somewhere.com, on Wed Feb 26 15:56:20 2014, using Rez v2.0.0 + + requested packages: + requests-2.2+ + python-2.6 + pymongo-0+<2.7 + + resolved packages: + python-2.6.8 /software/ext/python/2.6.8 + platform-linux /software/ext/platform/linux + requests-2.2.1 /software/ext/requests/2.2.1/python-2.6 + pymongo-2.6.3 /software/ext/pymongo/2.6.3 + arch-x86_64 /software/ext/arch/x86_64 + + > ]$ _ + +This example creates an environment containing the package 'houdini' version 12.5 +or greater, and runs the command 'hescape -h' inside that environment: + + ]$ rez-env houdini-12.5+ -- hescape -h + Usage: hescape [-foreground] [-s editor] [filename ...] + -h: output this usage message + -s: specify starting desktop by name + -foreground: starts process in foreground + +Resolved environments can also be created via the API: + + >>> import subprocess + >>> from rez.resolved_context import ResolvedContext + >>> + >>> r = ResolvedContext(["houdini-12.5+", "houdini-0+<13", "java", "!java-1.8+"]) + >>> p = r.execute_shell(command='which hescape', stdout=subprocess.PIPE) + >>> out, err = p.communicate() + >>> + >>> print out + '/software/ext/houdini/12.5.562/bin/hescape' diff --git a/wiki/pages/Notes.md b/wiki/pages/Notes.md new file mode 100644 index 000000000..613fe1e41 --- /dev/null +++ b/wiki/pages/Notes.md @@ -0,0 +1,4 @@ + + +## Bez Deprecation + diff --git a/wiki/pages/_Command-Line-Tools.md b/wiki/pages/_Command-Line-Tools.md new file mode 100644 index 000000000..e268e0a0b --- /dev/null +++ b/wiki/pages/_Command-Line-Tools.md @@ -0,0 +1,3 @@ +Auto-generated from __WIKI_PY_URL__ + +__GENERATED_MD__ \ No newline at end of file diff --git a/wiki/pages/_Configuring-Rez.md b/wiki/pages/_Configuring-Rez.md index 9368671f4..e45fae8d8 100644 --- a/wiki/pages/_Configuring-Rez.md +++ b/wiki/pages/_Configuring-Rez.md @@ -2,7 +2,7 @@ Rez has a good number of configurable settings. The default settings, and documentation for every setting, can be found -[here](https://github.com/nerdvegas/rez/blob/master/src/rez/rezconfig.py). +[here](https://github.com/__GITHUB_REPO__/blob/master/src/rez/rezconfig.py). Settings are determined in the following way: @@ -102,7 +102,7 @@ Here is an example showing how to override settings using your own configuration Following is an alphabetical list of rez settings. > [[media/icons/info.png]] Note that this list has been generated automatically -> from the [rez-config.py](https://github.com/nerdvegas/rez/blob/master/src/rez/rezconfig.py) +> from the [rez-config.py](https://github.com/__GITHUB_REPO__/blob/master/src/rez/rezconfig.py) > file in the rez source, so you can also refer to that file for the same information. __REZCONFIG_MD__ diff --git a/wiki/pages/__Footer.md b/wiki/pages/__Footer.md new file mode 100644 index 000000000..07d2dd923 --- /dev/null +++ b/wiki/pages/__Footer.md @@ -0,0 +1,4 @@ +[![Google Group](https://img.shields.io/badge/rez--config-Google%20Group-blue?style=flat-square&logo=google)](https://groups.google.com/forum/#!forum/rez-config) +[![Contributing Guidelines](https://img.shields.io/badge/rez-Contributing%20Guidelines-0b610e?style=flat-square&logo=github)](https://github.com/__GITHUB_REPO__/blob/master/CONTRIBUTING.md) +[![Report A Bug](https://img.shields.io/badge/rez-Report%20A%20Bug-critical?style=flat-square&logo=github)](https://github.com/__GITHUB_REPO__/issues/new) +[![Slack](https://img.shields.io/badge/slack-rez--talk-7a6800?style=flat-square&logo=slack)](https://rez-talk.slack.com/) diff --git a/wiki/pages/__Sidebar.md b/wiki/pages/__Sidebar.md new file mode 100644 index 000000000..a02400609 --- /dev/null +++ b/wiki/pages/__Sidebar.md @@ -0,0 +1,49 @@ +[![][wiki-badge]][wiki-actions] + +[wiki-badge]: https://github.com/__GITHUB_REPO__/workflows/__WORKFLOW__/badge.svg?branch=__BRANCH__ +[wiki-actions]: https://github.com/__GITHUB_REPO__/actions?query=workflow%3A__WORKFLOW__+branch%3A__BRANCH__ + +- [[Home]] + +:beginner: Introduction: + +- [[Installation]] and [[Pip]] +- [[Getting Started]] +- [[Basic Concepts]] +- [[Configuring Rez]] + +:memo: `package.py`: + +- [[Package Commands]] +- [[Package Definition Guide]] +- [[Variants]] + +:rocket: rez: + +- [[Contexts]] +- [[Suites]] +- [[Building Packages]] +- [[Environment Variables]] +- [[Command Line Tools]] +- [Python API](https://___GITHUB_USER___.github.io/__REPO_NAME__/) :link: + +:information_source: Others: + +- [[Glossary]] +- [[FAQ]] +- [[Notes]] +- [[Credits]] + +
:construction: Unwritten pages: + +- [[Advanced Topics]] +- [[Caching]] +- [[Releasing Packages]] +- [[Package Filters]] +- [[Testing Packages]] +- [[Rez GUI]] +- [[The Resolve Graph]] +- [[Timestamping]] +- [[Troubleshooting]] + +
\ No newline at end of file diff --git a/wiki/update-wiki.py b/wiki/update-wiki.py new file mode 100644 index 000000000..1ec6eae3c --- /dev/null +++ b/wiki/update-wiki.py @@ -0,0 +1,827 @@ +# -*- coding: utf-8 -*- +"""Python implementation of old ``update-wiki.sh`` merged with ``process.py``. + +*From ``update-wiki.sh``* + +This script calls git heavily to: +1. Takes the content from this repo; +2. Then writes it into a local clone of https://github.com/nerdvegas/rez.wiki.git; +3. Then follows the procedure outlined in README from 2. + +This process exists because GitHub does not support contributions to wiki +repositories - this is a workaround. + +See Also: + Original wiki update script files: + + - ``wiki/update-wiki.sh`` at rez 2.50.0, which calls + - ``utils/process.py`` from nerdvegas/rez.wiki at d632328, and + - ``utils/update.sh`` from nerdvegas/rez.wiki at d632328 +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +from collections import defaultdict +from io import open +import inspect +import os +import re +import subprocess +import shutil +import sys + + +ORIGINAL_getsourcefile = inspect.getsourcefile +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +REZ_SOURCE_DIR = os.getenv("REZ_SOURCE_DIR", os.path.dirname(THIS_DIR)) + +TMP_NAME = ".rez-gen-wiki-tmp" # See also: .gitignore +TEMP_WIKI_DIR = os.getenv("TEMP_WIKI_DIR", os.path.join(THIS_DIR, TMP_NAME)) +GITHUB_REPO = os.getenv("GITHUB_REPOSITORY", "nerdvegas/rez") +GITHUB_BRANCH = os.getenv("GITHUB_BRANCH", "master") +GITHUB_WORKFLOW = os.getenv("GITHUB_WORKFLOW", "Wiki") +CLONE_URL = os.getenv( + "CLONE_URL", + "git@github.com:{0}.wiki.git".format(GITHUB_REPO) +) + + +def PATCHED_getsourcefile(obj): + """Patch to not return None if path from inspect.getfile is not absolute. + + Returns: + str: Full path to source code file for an object, else this file path. + """ + return ORIGINAL_getsourcefile(obj) or THIS_FILE + + +################################################################################ +# https://github.com/rasbt/markdown-toclify +################################################################################ + +VALIDS = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-&' + + +def read_lines(in_file): + """Returns a list of lines from a input markdown file.""" + + with open(in_file, 'r') as inf: + in_contents = inf.read().split('\n') + return in_contents + + +def remove_lines(lines, remove=('[[back to top]', ' tags.""" + + if not remove: + return lines[:] + + out = [] + for l in lines: + if l.startswith(remove): + continue + out.append(l) + return out + + +def dashify_headline(line): + """ + Takes a header line from a Markdown document and + returns a tuple of the + '#'-stripped version of the head line, + a string version for anchor tags, + and the level of the headline as integer. + E.g., + >>> dashify_headline('### some header lvl3') + ('Some header lvl3', 'some-header-lvl3', 3) + """ + stripped_right = line.rstrip('#') + stripped_both = stripped_right.lstrip('#') + level = len(stripped_right) - len(stripped_both) + stripped_wspace = stripped_both.strip() + + # character replacements + replaced_colon = stripped_wspace.replace('.', '') + replaced_slash = replaced_colon.replace('/', '') + rem_nonvalids = ''.join([c if c in VALIDS + else '-' for c in replaced_slash]) + + lowered = rem_nonvalids.lower() + dashified = re.sub(r'(-)\1+', r'\1', lowered) # remove duplicate dashes + dashified = dashified.strip('-') # strip dashes from start and end + + # exception '&' (double-dash in github) + dashified = dashified.replace('-&-', '--') + + return [stripped_wspace, dashified, level] + + +def tag_and_collect(file_lines, id_tag=True, back_links=False, exclude_h=None): + """ + Gets headlines from the markdown document and creates anchor tags. + Keyword arguments: + lines: a list of sublists where every sublist + represents a line from a Markdown document. + id_tag: if true, creates inserts a the tags (not req. by GitHub) + back_links: if true, adds "back to top" links below each headline + exclude_h: header levels to exclude. E.g., [2, 3] + excludes level 2 and 3 headings. + Returns a tuple of 2 lists: + 1st list: + A modified version of the input list where + anchor tags where inserted + above the header lines (if github is False). + 2nd list: + A list of 3-value sublists, where the first value + represents the heading, the second value the string + that was inserted assigned to the IDs in the anchor tags, + and the third value is an integer that reprents the headline level. + E.g., + [['some header lvl3', 'some-header-lvl3', 3], ...] + """ + out_contents = [] + headlines = [] + + for line in file_lines: + saw_headline = False + orig_len = len(line) + + if re.match(r'^\#{1,6} ', line): + line = line.lstrip() + + # comply with new markdown standards + + # not a headline if '#' not followed by whitespace '##no-header': + if not line.lstrip('#').startswith(' '): + continue + # not a headline if more than 6 '#': + if len(line) - len(line.lstrip('#')) > 6: + continue + # headers can be indented by at most 3 spaces: + if orig_len - len(line) > 3: + continue + + # ignore empty headers + if not set(line) - {'#', ' '}: + continue + + saw_headline = True + dashified = dashify_headline(line) + + if not exclude_h or not dashified[-1] in exclude_h: + if id_tag: + id_tag = '' + out_contents.append(id_tag.format(dashified=dashified)) + headlines.append(dashified) + + out_contents.append(line) + if back_links and saw_headline: + out_contents.append('[[back to top](#table-of-contents)]') + return out_contents, headlines + + +def positioning_headlines(headlines): + """ + Strips unnecessary whitespaces/tabs if first header is not left-aligned + """ + left_just = False + for row in headlines: + if row[-1] == 1: + left_just = True + break + if not left_just: + for row in headlines: + row[-1] -= 1 + return headlines + + +def create_toc(headlines, hyperlink=True, top_link=False, no_toc_header=False): + """ + Creates the table of contents from the headline list + that was returned by the tag_and_collect function. + Keyword Arguments: + headlines: list of lists + e.g., ['Some header lvl3', 'some-header-lvl3', 3] + hyperlink: Creates hyperlinks in Markdown format if True, + e.g., '- [Some header lvl1](#some-header-lvl1)' + top_link: if True, add a id tag for linking the table + of contents itself (for the back-to-top-links) + no_toc_header: suppresses TOC header if True. + Returns a list of headlines for a table of contents + in Markdown format, + e.g., [' - [Some header lvl3](#some-header-lvl3)', ...] + """ + processed = [] + if not no_toc_header: + if top_link: + processed.append('\n') + processed.append('# Table of Contents') + + for line in headlines: + indent = (line[2] - 1) * ' ' + if hyperlink: + item = '%s- [%s](#%s)' % (indent, line[0], line[1]) + else: + item = '%s- %s' % (indent, line[0]) + processed.append(item) + processed.append('\n') + return processed + + +def build_markdown(toc_headlines, body, spacer=0, placeholder=None): + """ + Returns a string with the Markdown output contents incl. + the table of contents. + Keyword arguments: + toc_headlines: lines for the table of contents + as created by the create_toc function. + body: contents of the Markdown file including + ID-anchor tags as returned by the + tag_and_collect function. + spacer: Adds vertical space after the table + of contents. Height in pixels. + placeholder: If a placeholder string is provided, the placeholder + will be replaced by the TOC instead of inserting the TOC at + the top of the document + """ + if spacer: + spacer_line = ['\n
\n' % (spacer)] + toc_markdown = "\n".join(toc_headlines + spacer_line) + else: + toc_markdown = "\n".join(toc_headlines) + + body_markdown = "\n".join(body).strip() + + if placeholder: + markdown = body_markdown.replace(placeholder, toc_markdown) + else: + markdown = toc_markdown + body_markdown + + return markdown + + +def output_markdown(markdown_cont, output_file): + """ + Writes to an output file if `outfile` is a valid path. + """ + if output_file: + with open(output_file, 'w') as out: + out.write(markdown_cont) + + +def markdown_toclify(input_file, output_file=None, github=False, + back_to_top=False, no_link=False, + no_toc_header=False, spacer=0, placeholder=None, + exclude_h=None): + """ Function to add table of contents to markdown files. + Parameters + ----------- + input_file: str + Path to the markdown input file. + output_file: str (defaul: None) + Path to the markdown output file. + github: bool (default: False) + Uses GitHub TOC syntax if True. + back_to_top: bool (default: False) + Inserts back-to-top links below headings if True. + no_link: bool (default: False) + Creates the table of contents without internal links if True. + no_toc_header: bool (default: False) + Suppresses the Table of Contents header if True + spacer: int (default: 0) + Inserts horizontal space (in pixels) after the table of contents. + placeholder: str (default: None) + Inserts the TOC at the placeholder string instead + of inserting the TOC at the top of the document. + exclude_h: list (default None) + Excludes header levels, e.g., if [2, 3], ignores header + levels 2 and 3 in the TOC. + Returns + ----------- + cont: str + Markdown contents including the TOC. + """ + raw_contents = read_lines(input_file) + cleaned_contents = remove_lines(raw_contents, remove=('[[back to top]', '\d+)\s+(?P.+)\s*$', + flags=re.MULTILINE | re.UNICODE + ) + + for match in regex.finditer(out): + author = match.group('author') + author_html = "%s
" % aliases.get(author.lower(), author) + contributors[author_html] += int(match.group('commits')) + + return '\n'.join( + author_html for author_html, commit_count in + sorted(contributors.items(), key=lambda x: x[1], reverse=True) + ) + + +def process_markdown_files(): + no_toc = [ + "Credits.md", + "Command-Line-Tools.md", + "Home.md", + "_Footer.md", + "_Sidebar.md", + ] + + pagespath = os.path.join(THIS_DIR, "pages") + + src_path = os.getenv("REZ_SOURCE_DIR") + if src_path is None: + print( + "Must provide REZ_SOURCE_DIR which points at root of " + "rez source clone", file=sys.stderr, + ) + sys.exit(1) + + def do_replace(filename, token_md): + srcfile = os.path.join(pagespath, "_%s.md" % filename) + destfile = os.path.join(TEMP_WIKI_DIR, "%s.md" % filename) + + # with open(srcfile) as f: + with open(srcfile, encoding='utf-8') as f: + txt = f.read() + + for token, md in token_md.items(): + txt = txt.replace(token, md) + + print("Writing ", destfile, "...", sep="") + with open(destfile, 'w', encoding='utf-8') as f: + f.write(txt) + + # generate markdown from rezconfig.py, add to _Configuring-Rez.md and write + # out to Configuring-Rez.md + filepath = os.path.join(src_path, "src", "rez", "rezconfig.py") + with open(filepath) as f: + txt = f.read() + + do_replace( + "Configuring-Rez", + { + "__REZCONFIG_MD__": convert_rezconfig_src_to_md(txt), + "__GITHUB_REPO__": GITHUB_REPO, + } + ) + + # generate markdown contributors list, add to _Credits.md and write out to + # Credits.md + md = create_contributors_md(src_path) + do_replace("Credits", {"__CONTRIBUTORS_MD__": md}) + + do_replace( + "Command-Line-Tools", + { + "__GENERATED_MD__": make_cli_markdown(src_path), + "__WIKI_PY_URL__": make_cli_source_link(), + } + ) + + do_replace("_Footer", {"__GITHUB_REPO__": GITHUB_REPO}) + + try: + from urllib import quote + except ImportError: + from urllib.parse import quote + user, repo_name = GITHUB_REPO.split('/') + do_replace( + "_Sidebar", + { + "__GITHUB_REPO__": GITHUB_REPO, + "___GITHUB_USER___": user, + "__REPO_NAME__": repo_name, + "__WORKFLOW__": quote(GITHUB_WORKFLOW, safe=""), + "__BRANCH__": quote(GITHUB_BRANCH, safe=""), + } + ) + + # process each md file: + # * adds TOC; + # * replaces short-form content links like '[[here:Blah.md#Header]]' with full form; + # * copies to the root dir. + # + skip_regex = r'^_(?!(Sidebar|Footer))|(?